--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:40:11.168931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1ab9e-567bbdc5cc-77rv9 to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d451b9f975397d namespace: kserve-ci-e2e-test resourceVersion: "22072" uid: 181137f6-a451-466c-9962-1ef287abd3a0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d451ba1ba32604 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: ea2dfebd-b433-4664-9a03-702c711b20a9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab9e} kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d451ba1d59da36 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 148a178b-0bb3-4b0e-bccc-f70656a61983 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab9e} kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: 'Created container: ensemble-graph-1ab9e' metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d451ba200d7112 namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: 1d230cd5-a9b8-45f0-8fec-a401bd0a890a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab9e} kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: Started container ensemble-graph-1ab9e metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d451ba20839d46 namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: 329fcd9c-376c-4128-b8cf-d2e805b13033 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab9e} kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:48:25Z" message: Stopping container ensemble-graph-1ab9e metadata: creationTimestamp: "2026-09-11T16:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:25Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d4522d2860e623 namespace: kserve-ci-e2e-test resourceVersion: "25859" uid: 1ca012b5-aa3c-4885-afca-09597ed0acfa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:48:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1ab9e} kind: Pod name: ensemble-graph-1ab9e-567bbdc5cc-77rv9 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 705d63cf-7639-433c-9ee1-9d85cdd2c318 kind: Event lastTimestamp: "2026-09-11T16:48:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:53Z" name: ensemble-graph-1ab9e-567bbdc5cc-77rv9.18d4522dbb16b813 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: e1b3f67f-73ae-44cc-af7b-4798c1b22972 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1ab9e-567bbdc5cc namespace: kserve-ci-e2e-test resourceVersion: "22060" uid: dd8e83d8-dcea-42f6-8df6-58c1abd3af67 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: 'Created pod: ensemble-graph-1ab9e-567bbdc5cc-77rv9' metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e-567bbdc5cc.18d451b9f909f182 namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: af3f0a52-aa76-434c-9b92-58ecfe139d6b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: b211a511-3bb4-4928-9575-9565f58f8766 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: Scaled up replica set ensemble-graph-1ab9e-567bbdc5cc from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e.18d451b9f84d1175 namespace: kserve-ci-e2e-test resourceVersion: "22061" uid: 576a47f5-5989-4f71-9efd-706b7eef3473 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: fba76d0c-e6f0-4e27-bd04-c00707a990e6 kind: Event lastTimestamp: "2026-09-11T16:40:11Z" message: InferenceGraph [ensemble-graph-1ab9e] is Ready metadata: creationTimestamp: "2026-09-11T16:40:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:11Z" name: ensemble-graph-1ab9e.18d451b9fcd7cd0a namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: 39990f81-8a57-4037-b1aa-e3975c3e3084 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: a593dc58-0445-4be8-9495-1f2cd0903721 kind: Event lastTimestamp: "2026-09-11T16:40:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:26Z" name: ensemble-graph-1ab9e.18d451bd7727318a namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 414234cd-696c-4e6b-b793-e2b0168105bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22059" uid: a593dc58-0445-4be8-9495-1f2cd0903721 kind: Event lastTimestamp: "2026-09-11T16:40:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:26Z" name: ensemble-graph-1ab9e.18d451bd7791d854 namespace: kserve-ci-e2e-test resourceVersion: "22210" uid: b339d741-bb23-4f31-abcd-1ab54e2cbefe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22209" uid: a593dc58-0445-4be8-9495-1f2cd0903721 kind: Event lastTimestamp: "2026-09-11T16:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:41Z" name: ensemble-graph-1ab9e.18d451c0f685f6ed namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: e131e960-60b0-422d-8ed4-5228ae7d488a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "22209" uid: a593dc58-0445-4be8-9495-1f2cd0903721 kind: Event lastTimestamp: "2026-09-11T16:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:41Z" name: ensemble-graph-1ab9e.18d451c0f6e828a8 namespace: kserve-ci-e2e-test resourceVersion: "22348" uid: c0c468ca-a87c-40b2-af36-8a27c3c048fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:00.872008Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21090" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-a9581-694df647d-nn95d to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:00Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451a99b6f466c namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 8ee1cc12-352f-407b-b1bf-4c7f0cc2aa13 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:01Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:01Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451a9bd656f62 namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: e34424e4-9786-493f-a9c0-13ec40dde7ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9581} kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:01Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451a9be92888c namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: 4cb0ae3e-0633-49bb-b339-bb95803ff0a4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9581} kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:01Z" message: 'Created container: ensemble-graph-a9581' metadata: creationTimestamp: "2026-09-11T16:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:01Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451a9c2cde4b7 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 65d6ea40-20e4-48ba-910e-ded5863f1bb6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9581} kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:01Z" message: Started container ensemble-graph-a9581 metadata: creationTimestamp: "2026-09-11T16:39:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:01Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451a9c3483fad namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 501dd653-2386-43b5-9d64-3b5f61d17dc6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9581} kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:10Z" message: Stopping container ensemble-graph-a9581 metadata: creationTimestamp: "2026-09-11T16:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:10Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451abf440e012 namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 49bece6d-2adc-48a0-aaa9-bcda7318711b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-a9581} kind: Pod name: ensemble-graph-a9581-694df647d-nn95d namespace: kserve-ci-e2e-test resourceVersion: "21093" uid: 5e34d8cf-5311-4868-9966-f0a1ff81e0e0 kind: Event lastTimestamp: "2026-09-11T16:39:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:38Z" name: ensemble-graph-a9581-694df647d-nn95d.18d451ac72cf1e92 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 561112a1-888a-4ac1-9d67-2acc07122af8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-a9581-694df647d namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 7fc67ae5-6460-4df9-bb8b-8e7c3a2641bc kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: 'Created pod: ensemble-graph-a9581-694df647d-nn95d' metadata: creationTimestamp: "2026-09-11T16:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: ensemble-graph-a9581-694df647d.18d451a99aef23ba namespace: kserve-ci-e2e-test resourceVersion: "21092" uid: 2208c593-ed14-4b67-85dd-dcb8872ad725 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-a9581 namespace: kserve-ci-e2e-test resourceVersion: "21085" uid: d46c2ef8-c8d2-4af3-a675-f04af41eadc5 kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: Scaled up replica set ensemble-graph-a9581-694df647d from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: ensemble-graph-a9581.18d451a99a3e1881 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: 66ce47c8-4514-4b7f-9cac-a2d86b0237d9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-a9581 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 35f629cd-4246-4f4b-9520-003aeb2988dd kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: InferenceGraph [ensemble-graph-a9581] is Ready metadata: creationTimestamp: "2026-09-11T16:39:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:00Z" name: ensemble-graph-a9581.18d451a99ff697ef namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 0baaf7ac-46bf-4558-a966-9a487c97e7bf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:11.234552Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21301" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1ab9e-predictor-d4c755658-thbsc to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac051729c3 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 07b67f27-f34d-401f-b2d9-db9e58036b77 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1ab9e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac1119d171 namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 39c26b67-692e-44dd-bd0c-0b995ea68be1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac4b112670 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 6f951766-1012-41cb-ab84-6e3cf9742ef8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac4c6a6898 namespace: kserve-ci-e2e-test resourceVersion: "21339" uid: 07ff2f4e-22e4-4e9e-beff-f326ce555943 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac4f571012 namespace: kserve-ci-e2e-test resourceVersion: "21340" uid: 59e49b45-f3cf-4219-bf30-7190fba84dfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac4fcd60b3 namespace: kserve-ci-e2e-test resourceVersion: "21341" uid: b11a630c-16b8-4e96-a1b0-0ddf290993cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac4fd75727 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 8858d3d9-f632-4442-9e85-76b440a8e15c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac52f6d677 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: 2bae5de4-0a47-48fd-a41e-a2a9a04b3b42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac5369dd13 namespace: kserve-ci-e2e-test resourceVersion: "21345" uid: e87304ec-8b8d-4586-ad3f-f29b5e80431b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:48:29Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:29Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d451ac8c8bf71a namespace: kserve-ci-e2e-test resourceVersion: "26014" uid: 0dbb7092-a328-4025-8f0f-74756961ac78 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d4522d3357ebe7 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: 9c213e5a-2c99-4cf9-b285-dcba9da2d6ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d4522d33586b8b namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 3b7eb51b-2536-4c5c-b837-07678ead4e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: 40e272de-bdc9-468b-bf8f-9e7d706abc0e kind: Event lastTimestamp: "2026-09-11T16:48:29Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": dial tcp 10.132.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:29Z" name: error-404-isvc-1ab9e-predictor-d4c755658-thbsc.18d4522e01218baf namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 27d829a9-8205-4ca2-acc6-ed774c471cca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1ab9e-predictor-d4c755658 namespace: kserve-ci-e2e-test resourceVersion: "21294" uid: 42956616-8498-4547-b7d7-fe3e240b2f68 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'Created pod: error-404-isvc-1ab9e-predictor-d4c755658-thbsc' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e-predictor-d4c755658.18d451ac0466fe9b namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 8f009f59-3e5e-40bd-9700-5eb042adfce9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: 8ef2be4d-4cb9-4a2c-9ed5-c50a8fe51838 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Scaled up replica set error-404-isvc-1ab9e-predictor-d4c755658 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e-predictor.18d451ac02ddde9e namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: fb6c9afa-d8e7-424c-9ba2-22701797537f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: a062c2f4-43fd-4d42-814f-44108901bd25 kind: Event lastTimestamp: "2026-09-11T16:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:56Z" name: error-404-isvc-1ab9e-predictor.18d451af8152dd12 namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 54b141c0-dfde-4fe0-90e7-70ccbe1824b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: a062c2f4-43fd-4d42-814f-44108901bd25 kind: Event lastTimestamp: "2026-09-11T16:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:56Z" name: error-404-isvc-1ab9e-predictor.18d451af81b6f0b0 namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: b5fd3688-577b-4ac3-8ef4-6a6bc4fa1dbe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: a062c2f4-43fd-4d42-814f-44108901bd25 kind: Event lastTimestamp: "2026-09-11T16:40:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:26Z" name: error-404-isvc-1ab9e-predictor.18d451b300597d34 namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: 96b72c0a-db8b-415a-a024-6c6c8aa8902d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: a062c2f4-43fd-4d42-814f-44108901bd25 kind: Event lastTimestamp: "2026-09-11T16:40:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:26Z" name: error-404-isvc-1ab9e-predictor.18d451b3009f758c namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 2eb6b1e5-87d8-42f4-90e2-150c392b0b0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: fa0b048e-0ca7-4e98-bc37-c8ba29448c54 kind: Event lastTimestamp: "2026-09-11T16:39:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:59Z" name: error-404-isvc-1ab9e.18d451ac02d3e554 namespace: kserve-ci-e2e-test resourceVersion: "21952" uid: 9d54ba73-5d1f-47c4-a3b7-a6cd04882680 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: fa0b048e-0ca7-4e98-bc37-c8ba29448c54 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-1ab9e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ab9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e.18d451ac0362a652 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: 8d4d6661-c3ec-4cb5-8254-540d248fdbc3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: fa0b048e-0ca7-4e98-bc37-c8ba29448c54 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-1ab9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-1ab9e.18d451ac03636579 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 320f3727-e778-4ca0-9c18-c8634c073d68 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21950" uid: fa0b048e-0ca7-4e98-bc37-c8ba29448c54 kind: Event lastTimestamp: "2026-09-11T16:39:59Z" message: InferenceService [error-404-isvc-1ab9e] is Ready metadata: creationTimestamp: "2026-09-11T16:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:59Z" name: error-404-isvc-1ab9e.18d451b74820bca6 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: 2a236e34-b1cc-4d1e-9e98-aa9229c9cd31 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:49:36.499570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26852" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:49:36Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523d99ca3a08 namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: 6207800e-f9f4-4959-a58c-bdf237bdf34e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dbc02990a namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 3be93d6e-b359-4172-801c-a9d998d3c057 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dbd294206 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 4e4d5b5f-5043-4686-ae69-0aa2511b9132 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dc00cfe58 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 94b1a24f-e743-443f-b5f4-5ce07677befb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dc07d27a0 namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 5ae8738a-b5ba-4b5b-98ca-29f0185b16fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dc0863eea namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: 95c0b272-6064-4071-8c27-4b9fe11c540d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dc397ed98 namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: 7fdcfde9-1b77-4ba8-b3cb-20b5d82c9d7e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523dc407f850 namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: ec33764b-88a3-4d0d-beae-11603bcc5540 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:50:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:14Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d4523e183bbe3a namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: ae68e7cb-b8fd-408c-8531-050e384f708b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:58:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:58:51Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d452bebdb38e87 namespace: kserve-ci-e2e-test resourceVersion: "30955" uid: 1047eb54-b21b-4474-b084-7cc4e44d7e3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: d766b2f0-9eec-46e8-a152-653641963131 kind: Event lastTimestamp: "2026-09-11T16:58:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:58:51Z" name: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s.18d452bebdb3f897 namespace: kserve-ci-e2e-test resourceVersion: "30956" uid: 4042e523-d8b4-4a50-b6fd-6e7ad6355bf7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-20f8f-predictor-65795d6ffd namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 2defc4cf-01d1-469d-ab19-d0f1ac318939 kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: 'Created pod: error-404-isvc-20f8f-predictor-65795d6ffd-lqj2s' metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:36Z" name: error-404-isvc-20f8f-predictor-65795d6ffd.18d4523d995401e1 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: bf485c38-5809-443a-bb75-725aa6bd7a73 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 3f8098c3-e789-4ebf-82d1-411d2d4fe899 kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Scaled up replica set error-404-isvc-20f8f-predictor-65795d6ffd from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:36Z" name: error-404-isvc-20f8f-predictor.18d4523d97ce2d78 namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: 976a4a40-3837-42bb-bf89-1c5ec9d1105b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: f8c6d419-593e-4dc4-86f8-768ff43d00f8 kind: Event lastTimestamp: "2026-09-11T16:49:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:51Z" name: error-404-isvc-20f8f-predictor.18d45241167261ea namespace: kserve-ci-e2e-test resourceVersion: "27044" uid: bd684d37-5ec8-4a67-9658-c59d8190a7bf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: f8c6d419-593e-4dc4-86f8-768ff43d00f8 kind: Event lastTimestamp: "2026-09-11T16:49:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:51Z" name: error-404-isvc-20f8f-predictor.18d4524116d4e064 namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: 03c3742c-b3e9-49f7-aabf-2f8eaae2d944 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: f8c6d419-593e-4dc4-86f8-768ff43d00f8 kind: Event lastTimestamp: "2026-09-11T16:51:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:51:06Z" name: error-404-isvc-20f8f-predictor.18d4524495912bdf namespace: kserve-ci-e2e-test resourceVersion: "27707" uid: ccc9c345-1d96-4fda-98bb-5261d71dcb49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:50:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: f8c6d419-593e-4dc4-86f8-768ff43d00f8 kind: Event lastTimestamp: "2026-09-11T16:51:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:51:06Z" name: error-404-isvc-20f8f-predictor.18d4524495fdabd5 namespace: kserve-ci-e2e-test resourceVersion: "27708" uid: 077599d0-23c6-4d05-a3b9-414558de8354 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20f8f namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: 00721d5d-12b4-4268-8d4f-9dca5106e00b kind: Event lastTimestamp: "2026-09-11T16:50:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:24Z" name: error-404-isvc-20f8f.18d4523d97d10cc4 namespace: kserve-ci-e2e-test resourceVersion: "27351" uid: f6cbf7b9-93f0-4c95-963c-753a6703927b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27349" uid: 00721d5d-12b4-4268-8d4f-9dca5106e00b kind: Event lastTimestamp: "2026-09-11T16:50:24Z" message: InferenceService [error-404-isvc-20f8f] is Ready metadata: creationTimestamp: "2026-09-11T16:50:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:24Z" name: error-404-isvc-20f8f.18d45248d3c0cb90 namespace: kserve-ci-e2e-test resourceVersion: "27350" uid: 33db2078-033a-47a8-9c65-4bd286b6ac6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:49:01.972994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-482a1-predictor-686985f998-wdxbb to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:49:01Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d452358fd86e8f namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: 987845d0-4bc9-41eb-9c87-1e09b128485d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26386" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235b209b0e2 namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: fee5a83f-287f-4eef-b836-7a372571725b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235b33def66 namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: dfff3711-8bcc-4200-964b-17165e7d3446 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235b70bc365 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: aeaeef72-220f-4495-9b2d-1eaf65cf4f3f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235b788cd51 namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 48e87e4c-7381-4a3b-ae16-0b4727c22df0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235b7951ee0 namespace: kserve-ci-e2e-test resourceVersion: "26416" uid: 43d42b9e-5f6a-4e3a-b15a-94fc95a3c526 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235ba87d08c namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: 4cd06629-a2bd-4f76-b585-8c8e57eabada reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235baffa29a namespace: kserve-ci-e2e-test resourceVersion: "26418" uid: aa446151-b174-4208-84be-06a133104c59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T16:49:39Z" message: 'Readiness probe failed: dial tcp 10.132.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:39Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45235e7767902 namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: e8756194-d5b7-47da-bcb1-05acd7a9ef7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T17:06:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:21Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d453278426fff9 namespace: kserve-ci-e2e-test resourceVersion: "34075" uid: 1fc989e3-9568-407a-a127-c85dbe0d06c4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-482a1-predictor-686985f998-wdxbb namespace: kserve-ci-e2e-test resourceVersion: "26377" uid: 839e821c-fa44-488b-ad3a-1754ded0cb23 kind: Event lastTimestamp: "2026-09-11T17:06:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:21Z" name: error-404-isvc-482a1-predictor-686985f998-wdxbb.18d45327842ca3e9 namespace: kserve-ci-e2e-test resourceVersion: "34076" uid: c7f93903-6952-41fb-8d6d-33c50a122162 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-482a1-predictor-686985f998 namespace: kserve-ci-e2e-test resourceVersion: "26367" uid: 91647e21-59e8-451f-a812-1525587548dd kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: 'Created pod: error-404-isvc-482a1-predictor-686985f998-wdxbb' metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:01Z" name: error-404-isvc-482a1-predictor-686985f998.18d452358eba4ffc namespace: kserve-ci-e2e-test resourceVersion: "26375" uid: 744cccd4-a6e2-482e-b350-94b89e63595c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 8dc5a3ed-cb0b-4ac5-af5c-cfb0f6fdb096 kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Scaled up replica set error-404-isvc-482a1-predictor-686985f998 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:01Z" name: error-404-isvc-482a1-predictor.18d452358af4bc5c namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: 9505beb1-2bf1-4ebe-ac45-33bb095fd1ef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: c6de4848-853e-4400-9242-1d5bf8eb3a1b kind: Event lastTimestamp: "2026-09-11T16:49:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:16Z" name: error-404-isvc-482a1-predictor.18d452390a7195bb namespace: kserve-ci-e2e-test resourceVersion: "26570" uid: 6603feba-2b72-48be-a3f3-69f25a11d43e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26366" uid: c6de4848-853e-4400-9242-1d5bf8eb3a1b kind: Event lastTimestamp: "2026-09-11T16:49:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:16Z" name: error-404-isvc-482a1-predictor.18d452390b164937 namespace: kserve-ci-e2e-test resourceVersion: "26572" uid: 7619ba64-9eb3-4b36-84bc-20b81215e971 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: c6de4848-853e-4400-9242-1d5bf8eb3a1b kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: error-404-isvc-482a1-predictor.18d4523c89c04adb namespace: kserve-ci-e2e-test resourceVersion: "27284" uid: 3ee94ff7-68de-47c0-9f6d-7957073eba66 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:49:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26571" uid: c6de4848-853e-4400-9242-1d5bf8eb3a1b kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: error-404-isvc-482a1-predictor.18d4523c8a2bd418 namespace: kserve-ci-e2e-test resourceVersion: "27285" uid: e15e7391-d33a-4c9c-8f43-6d938d41b1d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-482a1 namespace: kserve-ci-e2e-test resourceVersion: "26352" uid: 3c4e544c-77dd-4521-a938-d526f78b8fbf kind: Event lastTimestamp: "2026-09-11T16:49:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:49Z" name: error-404-isvc-482a1.18d452358afc8be4 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 2c504bd8-90ff-4fd4-8d32-d5d785eb7cf3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 3c4e544c-77dd-4521-a938-d526f78b8fbf kind: Event lastTimestamp: "2026-09-11T16:49:49Z" message: InferenceService [error-404-isvc-482a1] is Ready metadata: creationTimestamp: "2026-09-11T16:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:49Z" name: error-404-isvc-482a1.18d45240a2820517 namespace: kserve-ci-e2e-test resourceVersion: "27024" uid: 0eb3b183-f58a-4177-8be0-dc36fa6e37d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:27:01.846646Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8975a-predictor-748969fd7d-k9kmm to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:27:01Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45102322ef8f3 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: d5e9715f-c760-417f-9ab2-45ad9c5dfc59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8975a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451023c72096f namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: 9789f3f1-5362-4295-9b26-8470073b4904 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14850" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:27:03Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45102ad940d2c namespace: kserve-ci-e2e-test resourceVersion: "14962" uid: a14d12f4-7ff6-43bc-8e96-8b42ab9d2399 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45102aedc3e8c namespace: kserve-ci-e2e-test resourceVersion: "14964" uid: 522a910f-288f-4221-a11a-349b957eeb7b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:15Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" in 11.719s (11.719s including waiting). Image size: 1357269580 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:15Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451056960f5e0 namespace: kserve-ci-e2e-test resourceVersion: "15073" uid: 77a09a8f-b5f5-4877-8057-4d3d46ad506d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:15Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451056cac9a1c namespace: kserve-ci-e2e-test resourceVersion: "15074" uid: 99d5c6b6-122e-414d-8672-52fbbc49731a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:15Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451056d1da213 namespace: kserve-ci-e2e-test resourceVersion: "15075" uid: 5a198b90-6296-4bf9-80ae-0a4997def159 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:15Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-11T16:27:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:15Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451056d2a086e namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: f8ba007f-d812-4b69-bdb6-9d69d0dfbef1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:17Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.185s (2.185s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:17Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45105ef7569be namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: c327d6bd-fd3b-4891-bc07-1aae79a28aba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:17Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45105f396ca11 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: b58d7c8b-5ad1-4194-8672-2a1001212223 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:17Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d45105f4072977 namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 4490f7d3-cae7-4a0f-8631-231f395d7754 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:27:54Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:54Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d451062f8480bd namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: 85370b23-1c10-4786-84df-412bba1622e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d4511a722240ee namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 63602e21-573a-4d7a-a3ef-1892e1a117e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 468bcf17-fbfc-488e-b25f-73bfb2fd3077 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-8975a-predictor-748969fd7d-k9kmm.18d4511a7222be8e namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: dff016a1-5a98-4cb4-a444-d4a2b2ccfb1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8975a-predictor-748969fd7d namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: 51ccdf01-a68f-4b24-82ac-4d141f3d6874 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: 'Created pod: error-404-isvc-8975a-predictor-748969fd7d-k9kmm' metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: error-404-isvc-8975a-predictor-748969fd7d.18d45102319f1091 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 8f1715c4-42c8-467c-a27e-6f79338fe992 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14829" uid: 43ea321e-391c-46f9-a710-6eb4be546f13 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: Scaled up replica set error-404-isvc-8975a-predictor-748969fd7d from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: error-404-isvc-8975a-predictor.18d45102301ef2cd namespace: kserve-ci-e2e-test resourceVersion: "14832" uid: c8ff2970-9fa9-4e00-a0f6-538ba30f4bef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 3f603115-5616-4292-9823-595243b6d188 kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: error-404-isvc-8975a-predictor.18d45105aea88d24 namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 8ab95839-0693-49fe-a24b-0d2ad35ba00a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 3f603115-5616-4292-9823-595243b6d188 kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: error-404-isvc-8975a-predictor.18d45105aeef9587 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: cd35d92b-1720-4252-b2c4-cd587ff19e8e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 3f603115-5616-4292-9823-595243b6d188 kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: error-404-isvc-8975a-predictor.18d451102b30036d namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 2b16ce8f-cc6c-47b3-aca6-9ab38fa64de0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 3f603115-5616-4292-9823-595243b6d188 kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: error-404-isvc-8975a-predictor.18d451102ba7f059 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 5f20ee96-7485-454e-a12a-7b9a0b5ec8e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8975a namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: f56fc277-a7dc-4817-94da-3725a1a412c7 kind: Event lastTimestamp: "2026-09-11T16:28:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:05Z" name: error-404-isvc-8975a.18d4510230240793 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: 3e6421cb-97d0-473b-ba55-9a169db916f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8975a namespace: kserve-ci-e2e-test resourceVersion: "15541" uid: f56fc277-a7dc-4817-94da-3725a1a412c7 kind: Event lastTimestamp: "2026-09-11T16:28:05Z" message: InferenceService [error-404-isvc-8975a] is Ready metadata: creationTimestamp: "2026-09-11T16:28:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:05Z" name: error-404-isvc-8975a.18d45110eb9a191c namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: 75176630-758e-49a7-8dd8-a360b35ff6f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:28:46.112691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511a78ec96e5 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 9a929a0a-19c3-4f69-bf34-ef008da20511 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9c92a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511a8198ab19 namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: b10169c4-b616-48ce-a5f6-a3e681d38824 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511abf071896 namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 4b63a3da-7fbf-44ed-a56a-0a89b1de87d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac05ad1d6 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 3f84f274-d5e0-474a-a8b6-077da5275f9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac3dedb0a namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 6920bd9f-64e2-4da7-8514-261b20089a7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac46d0663 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: ac263ebe-e90b-42fe-9696-0e946d7c27ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac4779bc8 namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 4d97adc8-56fc-48bf-84a4-460db9853b02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac7f5cf4c namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: f58a40b7-f171-41cf-bdd7-3b79c7e6acb0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:28:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:47Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511ac86977fc namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 4676990a-fdc2-4811-8bae-1621831fdb37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:28:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:29:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:28:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:24Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4511b01ace168 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: d4c53389-5ecd-4843-b665-c4201960dc9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4519ba1b30d67 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 52a2af81-e661-4c5f-bd5f-8678c6872c30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6 namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 5fd0e389-c715-4715-b7f3-ae4821860541 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6.18d4519ba1b3a974 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 68530ae3-f721-40a0-b56e-faaddd21ec95 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9c92a-predictor-7cc7d7578 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: d46f05ec-1b5e-49e4-a104-3bd8330cfccf kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'Created pod: error-404-isvc-9c92a-predictor-7cc7d7578-x4tw6' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a-predictor-7cc7d7578.18d4511a7876c5e7 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 21a32cc5-32f1-4646-9775-354cccaa34af reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: aa4fa8fb-d12c-44ae-8904-265fcbe3a7e3 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Scaled up replica set error-404-isvc-9c92a-predictor-7cc7d7578 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a-predictor.18d4511a75a6c149 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: d9d176c3-d5ff-4e95-92da-831cca280e10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: b15350ef-8bac-4842-8f86-b8968c3bbc4d kind: Event lastTimestamp: "2026-09-11T16:29:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:16Z" name: error-404-isvc-9c92a-predictor.18d4511df47a9642 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: f9cad474-f4a2-43be-9bd6-ada2a862cea5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: b15350ef-8bac-4842-8f86-b8968c3bbc4d kind: Event lastTimestamp: "2026-09-11T16:29:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:16Z" name: error-404-isvc-9c92a-predictor.18d4511df5703c05 namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 8ba2801d-2b70-4384-984e-a0e2d62b0c02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: b15350ef-8bac-4842-8f86-b8968c3bbc4d kind: Event lastTimestamp: "2026-09-11T16:29:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:46Z" name: error-404-isvc-9c92a-predictor.18d45124f409ac3c namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: 04ceab3e-5265-4abd-a8d8-3b2fee70dd7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: b15350ef-8bac-4842-8f86-b8968c3bbc4d kind: Event lastTimestamp: "2026-09-11T16:29:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:46Z" name: error-404-isvc-9c92a-predictor.18d45124f46dac85 namespace: kserve-ci-e2e-test resourceVersion: "16827" uid: 13372923-e471-48ed-96da-c8b1567b206e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 45b21567-521a-4a83-a570-751ec09b0be0 kind: Event lastTimestamp: "2026-09-11T16:29:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:34Z" name: error-404-isvc-9c92a.18d4511a75a290f0 namespace: kserve-ci-e2e-test resourceVersion: "16701" uid: d90bfa36-050a-42e5-b929-f820aa5ea65f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 45b21567-521a-4a83-a570-751ec09b0be0 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-9c92a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c92a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a.18d4511a76c05fb9 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 751f00bd-fef9-4897-b58c-bc5f24fb5e0e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: 45b21567-521a-4a83-a570-751ec09b0be0 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-9c92a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:46Z" name: error-404-isvc-9c92a.18d4511a76c0b962 namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 31085ce8-9a2f-4971-bc8f-b694414127bc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16695" uid: 45b21567-521a-4a83-a570-751ec09b0be0 kind: Event lastTimestamp: "2026-09-11T16:29:34Z" message: InferenceService [error-404-isvc-9c92a] is Ready metadata: creationTimestamp: "2026-09-11T16:29:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:34Z" name: error-404-isvc-9c92a.18d45125bd1f18a0 namespace: kserve-ci-e2e-test resourceVersion: "16696" uid: 2d48979d-3756-4bc3-93a2-04a805f848f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:38:00.998688Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a9581-predictor-74b4589474-bz7h6 to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519baab4f4d9 namespace: kserve-ci-e2e-test resourceVersion: "20408" uid: bc541de8-9bb2-4398-bf7e-7331efeb2f61 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bcccbb0cd namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 431c820b-dafb-4708-aef4-62fcb9f0d6a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bce0c8957 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: c3498ebb-2087-416a-88da-9cf5d2060302 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bd0f05ebf namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: c8bed843-a7d6-45d7-9eb2-75cf5e5d4fd2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bd15f0bc0 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: d00dc473-b8ae-4b85-9f0e-8b75dc0b451b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bd169249e namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: d3935323-b26e-4410-896a-2fc348f447cf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bd4527009 namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: c4c1b4bb-ce7e-4674-9d5e-173f17421b7f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519bd4c0e261 namespace: kserve-ci-e2e-test resourceVersion: "20434" uid: f0d704fd-b62b-4cbb-a4f9-c5b0b1a2915b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d4519beebb30b7 namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: 29509899-d42b-454d-96e4-b45417a2c93f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d451abfbaeceb4 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: e5f65dc5-6799-443c-bf70-110dd05b2574 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d451abfbaf85ec namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: c58063f8-3342-41be-aa2b-45d7c8d217d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9581-predictor-74b4589474-bz7h6 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: d3c9a3e5-4152-48bb-97bf-dbca21b261fa kind: Event lastTimestamp: "2026-09-11T16:39:13Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:13Z" name: error-404-isvc-a9581-predictor-74b4589474-bz7h6.18d451ac76edf8b9 namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 422085e8-0e57-4b2a-aa58-6d75885531c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a9581-predictor-74b4589474 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: fdfb6f43-4a35-41b7-a78c-bb4d365b4bd7 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'Created pod: error-404-isvc-a9581-predictor-74b4589474-bz7h6' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-a9581-predictor-74b4589474.18d4519ba9cf14be namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 66209d0c-a1cc-4650-8b61-4758818e1880 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: f93ccf47-a02d-42e3-a264-342355c42eee kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Scaled up replica set error-404-isvc-a9581-predictor-74b4589474 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-a9581-predictor.18d4519ba53a75d3 namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: 84020618-bb6f-45cb-9edf-d32d7c9086ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 91fc941c-c259-4394-8316-01c298369a30 kind: Event lastTimestamp: "2026-09-11T16:38:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:30Z" name: error-404-isvc-a9581-predictor.18d4519f22fe459b namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: a30d7227-ee01-408e-819b-40e462108a80 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 91fc941c-c259-4394-8316-01c298369a30 kind: Event lastTimestamp: "2026-09-11T16:38:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:30Z" name: error-404-isvc-a9581-predictor.18d4519f235d58c3 namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: e15190ff-aad5-472f-be68-ddfa1c0b466c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 91fc941c-c259-4394-8316-01c298369a30 kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: error-404-isvc-a9581-predictor.18d451a620bc0128 namespace: kserve-ci-e2e-test resourceVersion: "21108" uid: b6bed896-46cf-4fec-a38b-d4f481db8ddb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20564" uid: 91fc941c-c259-4394-8316-01c298369a30 kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: error-404-isvc-a9581-predictor.18d451a621209f9b namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 3e64eed8-8c27-4f1d-a914-40972255a847 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: d4d49390-1e81-476a-b955-cda7c8905954 kind: Event lastTimestamp: "2026-09-11T16:38:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:48Z" name: error-404-isvc-a9581.18d4519ba508a0df namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: f1e01a87-a478-4801-aa74-6e06fff040f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: d4d49390-1e81-476a-b955-cda7c8905954 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-a9581": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-a9581.18d4519ba6df6897 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: e1766c79-d12d-45b3-997a-2c5cef04f95b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: d4d49390-1e81-476a-b955-cda7c8905954 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:00Z" name: error-404-isvc-a9581.18d4519ba6dfb4a5 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: eba932f5-578b-4d62-814d-1b6b00cff95a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: d4d49390-1e81-476a-b955-cda7c8905954 kind: Event lastTimestamp: "2026-09-11T16:38:48Z" message: InferenceService [error-404-isvc-a9581] is Ready metadata: creationTimestamp: "2026-09-11T16:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:48Z" name: error-404-isvc-a9581.18d451a6aa03b646 namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: d6ccff91-b653-4e96-a4a8-c10369730779 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:48:26.150190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25939" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ddc34-predictor-64558dffdb-gvgtl to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d38a3ddd4 namespace: kserve-ci-e2e-test resourceVersion: "25943" uid: c227a0ba-f907-40de-9002-2bfca4c9aa8d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ddc34-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d4025a3ae namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: 070c097e-a095-4772-b217-d709d7a8a200 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d7eb47d7d namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 17b8e337-f7fe-44b5-a128-3f21aa90e380 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d7ff7bf80 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: c4e6d3aa-a93a-403b-bc6c-4ecfea682080 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d8433c3b9 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: c2ab1759-e185-42ab-aafe-0c27ab39029e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d84ba2b40 namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: 09964432-03c1-4274-8963-626f153e3278 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d84c923c2 namespace: kserve-ci-e2e-test resourceVersion: "25982" uid: b0492125-e27c-43ef-8d64-7652486a44b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d88e8b84a namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: ef61c38f-3b91-4dab-a58e-3dbcdb780cd1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522d896a8a53 namespace: kserve-ci-e2e-test resourceVersion: "25985" uid: edbf9fc6-dfa5-49eb-8ffc-c3dcb1b4e96c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:48:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:49:05Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:48:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:05Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4522df20c5cf1 namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: b1b400c4-7455-4cf1-83e3-40849bda86bd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:36Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4523d8ecb2dec namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 5f6860ab-49c6-4d6d-9f4a-48402859619c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: 6a436a79-c6a0-4ae0-bcc7-d77a3e7f7ee5 kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:36Z" name: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl.18d4523d8ecbd222 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: 48a38507-ecc3-452d-b1b5-76d1122dbe77 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ddc34-predictor-64558dffdb namespace: kserve-ci-e2e-test resourceVersion: "25931" uid: 2e3aa14e-4c3d-44c0-859a-ed9bd0dbdce3 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'Created pod: error-404-isvc-ddc34-predictor-64558dffdb-gvgtl' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34-predictor-64558dffdb.18d4522d382d1dbe namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: e9d52411-c5ad-482d-95c5-0e3d56792230 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25924" uid: d45b841e-f1e6-435e-9778-13545066a207 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: Scaled up replica set error-404-isvc-ddc34-predictor-64558dffdb from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34-predictor.18d4522d36906310 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: 3102d139-79db-4ab0-b981-5bfc23befce5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 4837be85-9d13-4c0f-a33e-9802f272ca0b kind: Event lastTimestamp: "2026-09-11T16:49:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:11Z" name: error-404-isvc-ddc34-predictor.18d45230b4f70eca namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: 37f2985c-87b6-4a91-90d5-165b62ef1383 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 4837be85-9d13-4c0f-a33e-9802f272ca0b kind: Event lastTimestamp: "2026-09-11T16:49:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:11Z" name: error-404-isvc-ddc34-predictor.18d45230b52f77ad namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 899501a0-43c3-40e3-8889-1fe77c79043c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 4837be85-9d13-4c0f-a33e-9802f272ca0b kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: error-404-isvc-ddc34-predictor.18d4523433f3a822 namespace: kserve-ci-e2e-test resourceVersion: "26660" uid: 8a05fe6d-9533-4a40-93dd-dae2890d4548 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 4837be85-9d13-4c0f-a33e-9802f272ca0b kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: error-404-isvc-ddc34-predictor.18d452343460d3aa namespace: kserve-ci-e2e-test resourceVersion: "26662" uid: 22ccc7ca-5769-4890-a4db-c44d9e3d75e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: fd4a475e-6422-4845-929a-d9e47c5435b2 kind: Event lastTimestamp: "2026-09-11T16:49:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:15Z" name: error-404-isvc-ddc34.18d4522d36956f20 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 0175f740-b8bd-40b6-af30-359db766722e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: fd4a475e-6422-4845-929a-d9e47c5435b2 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-ddc34": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ddc34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34.18d4522d3714f59e namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 4ee2f715-b723-4dc2-b261-c0f6675f07f9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: fd4a475e-6422-4845-929a-d9e47c5435b2 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ddc34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:48:26Z" name: error-404-isvc-ddc34.18d4522d371548e6 namespace: kserve-ci-e2e-test resourceVersion: "25937" uid: ef092b02-283a-490a-a32b-25745c41bb75 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: fd4a475e-6422-4845-929a-d9e47c5435b2 kind: Event lastTimestamp: "2026-09-11T16:49:15Z" message: InferenceService [error-404-isvc-ddc34] is Ready metadata: creationTimestamp: "2026-09-11T16:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:15Z" name: error-404-isvc-ddc34.18d45238ad63ba19 namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 3cc157bc-ca55-4faa-baf2-c8771ca0d4ad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:38:36.840669Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20812" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:38:36Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a4030e2b10 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 1788108e-7f98-404e-9d19-601e007f968a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e5408-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:37Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a40e480664 namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: a8908865-9b3d-48f9-8d77-2cd46154a0ca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a44900c75a namespace: kserve-ci-e2e-test resourceVersion: "20851" uid: 010889a5-a17a-4c89-906d-c1c157105712 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a44a53797a namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 7e09afe3-e606-44de-a4bb-8e469a6f2fee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a44d775356 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: e64b0bbf-2322-4457-8a27-3b793762d242 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a44deada77 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: 1f7f1cf7-2ca0-4e48-9e98-6e2f999f15b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a44df4c93a namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 4f472328-3c9e-4e95-af65-5667b3b48838 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a4518c374f namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 5973a27f-6e3f-4055-8a64-a6120dad1646 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a4520336bc namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: 1ddea5c9-ea2a-42f0-a325-702b488a7166 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:39:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:14Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451a45b74472b namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 5996420c-8c2b-4043-92eb-d184e8f7cceb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:46Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451b452778946 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: 603beec1-8598-4e86-9570-e0779a878e98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:46Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451b452780c6e namespace: kserve-ci-e2e-test resourceVersion: "21750" uid: e17a6ab7-5722-4985-8640-791aa8b7a52d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6 namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: fe5ca859-4a2a-4551-add1-58c67ffbfd7e kind: Event lastTimestamp: "2026-09-11T16:39:49Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:49Z" name: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6.18d451b4e3b2b6a4 namespace: kserve-ci-e2e-test resourceVersion: "21849" uid: eeeb82ca-9257-4060-bce1-cdf5b7062fe3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e5408-predictor-7cfd9ff86 namespace: kserve-ci-e2e-test resourceVersion: "20806" uid: 246c2160-4b4b-4ecf-8bf8-7abf38c10082 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: 'Created pod: error-404-isvc-e5408-predictor-7cfd9ff86-4zpt6' metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:36Z" name: error-404-isvc-e5408-predictor-7cfd9ff86.18d451a402a6de58 namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 3695913b-88aa-46e7-ba38-a2b9596ae3df reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20804" uid: 07b0046b-d639-48a8-9e42-24b85c673a62 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Scaled up replica set error-404-isvc-e5408-predictor-7cfd9ff86 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:36Z" name: error-404-isvc-e5408-predictor.18d451a400c46a9a namespace: kserve-ci-e2e-test resourceVersion: "20807" uid: 7e2ad8db-c8f7-4608-af7a-887220f5b07d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 1c737b37-8c24-4465-9640-551a1726cfdc kind: Event lastTimestamp: "2026-09-11T16:39:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:06Z" name: error-404-isvc-e5408-predictor.18d451a77f481d06 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 6e3dfb83-fe44-4f94-8057-26e83ccb574f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: 1c737b37-8c24-4465-9640-551a1726cfdc kind: Event lastTimestamp: "2026-09-11T16:39:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:06Z" name: error-404-isvc-e5408-predictor.18d451a77fb5914a namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 5e4f18aa-06ab-4e08-9842-1be5c9482326 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 1c737b37-8c24-4465-9640-551a1726cfdc kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: error-404-isvc-e5408-predictor.18d451ae7d165caa namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 19d662c0-e4cf-4ae2-9d4c-a06c31888ad0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "21012" uid: 1c737b37-8c24-4465-9640-551a1726cfdc kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: error-404-isvc-e5408-predictor.18d451ae7d595eb5 namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 25df362c-636a-48c4-bae1-0093b10dc01c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5408 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 5c1e4a43-c6ac-469c-b5da-fe73afee2f2c kind: Event lastTimestamp: "2026-09-11T16:39:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:24Z" name: error-404-isvc-e5408.18d451a400ca95b4 namespace: kserve-ci-e2e-test resourceVersion: "21463" uid: 28550f20-c924-4a43-b9cf-ca986d3878d2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5408 namespace: kserve-ci-e2e-test resourceVersion: "21461" uid: 5c1e4a43-c6ac-469c-b5da-fe73afee2f2c kind: Event lastTimestamp: "2026-09-11T16:39:24Z" message: InferenceService [error-404-isvc-e5408] is Ready metadata: creationTimestamp: "2026-09-11T16:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:24Z" name: error-404-isvc-e5408.18d451af16e96ed3 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 4f8c1fb4-625f-46e2-a189-d56bf388b4b2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:47.096171Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ea3c5-predictor-848549444-hb7jc to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b45e9c18a8 namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 47f1e651-b153-49b3-aad3-e97c2c95df69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21800" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b480bedc8b namespace: kserve-ci-e2e-test resourceVersion: "21813" uid: 096d6c0b-3a92-4cf6-9b2c-b277bc32374f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b482001315 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 2827f3fd-feae-4205-9b91-8220d69bc4ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b484ffe62b namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: 07a4c835-be2f-4531-b3fb-8bb6021bef55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b485763531 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: ed1f5820-0889-4bda-b20b-c441c8449da2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b48581a96c namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 316c49e8-86d7-433b-82e3-01bff04562f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b4888815bd namespace: kserve-ci-e2e-test resourceVersion: "21820" uid: ff143c83-a7bc-4609-b370-5a827096f86a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b48910a973 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 8581927c-73de-4830-8e48-69f34b961855 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:40:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:25Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d451b4f9dc4b51 namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 889728cc-1dbb-4512-8cee-9dd28e5243d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:01Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d45235827ebf21 namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: 158af39e-924c-48ba-8233-b08b046effd2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ea3c5-predictor-848549444-hb7jc namespace: kserve-ci-e2e-test resourceVersion: "21791" uid: da08659d-2d8b-4597-9167-66d3e9859aad kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:01Z" name: error-404-isvc-ea3c5-predictor-848549444-hb7jc.18d452358281acb0 namespace: kserve-ci-e2e-test resourceVersion: "26334" uid: 77f27a38-d698-48fe-9829-f37487bf5246 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ea3c5-predictor-848549444 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: b3e31c12-f0c1-4234-997a-8a3eed02456b kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: 'Created pod: error-404-isvc-ea3c5-predictor-848549444-hb7jc' metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor-848549444.18d451b45e14c478 namespace: kserve-ci-e2e-test resourceVersion: "21790" uid: 80027544-c18e-452f-a7f4-fd23e670874d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: 5783684e-39b3-4858-a5ee-6e30f58b253a kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: Scaled up replica set error-404-isvc-ea3c5-predictor-848549444 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:47Z" name: error-404-isvc-ea3c5-predictor.18d451b45c5df1fa namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: 3daf16a8-eb61-4a04-adc4-145d9c7d6908 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: d74703c2-0894-423e-8790-67320c520b8d kind: Event lastTimestamp: "2026-09-11T16:40:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:17Z" name: error-404-isvc-ea3c5-predictor.18d451b7dae26b25 namespace: kserve-ci-e2e-test resourceVersion: "22140" uid: 5fb73bf5-9521-4504-b314-a46af8efbf67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: d74703c2-0894-423e-8790-67320c520b8d kind: Event lastTimestamp: "2026-09-11T16:40:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:17Z" name: error-404-isvc-ea3c5-predictor.18d451b7db4b7903 namespace: kserve-ci-e2e-test resourceVersion: "22141" uid: 048cd17e-8245-42ec-8469-3bc62906015b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: d74703c2-0894-423e-8790-67320c520b8d kind: Event lastTimestamp: "2026-09-11T16:41:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:41:02Z" name: error-404-isvc-ea3c5-predictor.18d451bed8c45b73 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: 914c4ca5-f13b-4cf9-a509-6b3b9534e88b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: d74703c2-0894-423e-8790-67320c520b8d kind: Event lastTimestamp: "2026-09-11T16:41:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:41:02Z" name: error-404-isvc-ea3c5-predictor.18d451bed92eb8eb namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 459bb4e1-a605-4f0a-a640-29bab5168504 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 5ce37277-28c3-476a-8088-281e3fdcb589 kind: Event lastTimestamp: "2026-09-11T16:40:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:35Z" name: error-404-isvc-ea3c5.18d451b45c668db2 namespace: kserve-ci-e2e-test resourceVersion: "22307" uid: a851db74-695a-46b5-8dc5-195266db406d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22305" uid: 5ce37277-28c3-476a-8088-281e3fdcb589 kind: Event lastTimestamp: "2026-09-11T16:40:35Z" message: InferenceService [error-404-isvc-ea3c5] is Ready metadata: creationTimestamp: "2026-09-11T16:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:35Z" name: error-404-isvc-ea3c5.18d451bfb5203a6e namespace: kserve-ci-e2e-test resourceVersion: "22306" uid: df10674c-5462-4d9d-abf0-76fdf551ba7b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:29:21.875268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ff919-predictor-754fd47c76-zdg46 to ip-10-0-128-17.ec2.internal metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:29:21Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122cc8a2f08 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 10500a71-c309-4120-90e6-4f7f546f1a46 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122eea20233 namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: e8b90b85-608f-4551-bafc-fba7da415bbb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122eff3827e namespace: kserve-ci-e2e-test resourceVersion: "16563" uid: abca535f-ea5a-4560-b614-7936d0ddcf31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122f2dfd3e3 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 1830854f-58d0-4927-944c-a7b63b1debb2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122f351ff8b namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: 24fe642e-b22c-41db-8e7f-bfd8fc55ea62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122f35b978a namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: a5dbee2c-d04b-4c12-92fa-a3cc8aee68e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122f5ec16c8 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: 95971106-e7fe-4447-ab58-2e73ebecdf74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d45122f65b23b8 namespace: kserve-ci-e2e-test resourceVersion: "16570" uid: 4d52cc0a-2c12-46ba-a7b6-54a6b3652c50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:30:00Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:30:00Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d451236e428c93 namespace: kserve-ci-e2e-test resourceVersion: "16953" uid: 4c109a71-857d-4ff0-80b5-5fe3067e9113 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d451a3fe0ec77b namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 0f408eb4-c599-4d5f-b9a4-9f3e00faf56b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ff919-predictor-754fd47c76-zdg46 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: b54c0a42-0d7b-4cee-9be8-154bb8d28ef9 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: error-404-isvc-ff919-predictor-754fd47c76-zdg46.18d451a3fe0f8c1a namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 55c7e421-30bd-48b0-919a-9bf2c52dc072 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-17.ec2.internal source: component: kubelet host: ip-10-0-128-17.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ff919-predictor-754fd47c76 namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 5b340346-bae2-4c98-842a-1a9f57947a5f kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'Created pod: error-404-isvc-ff919-predictor-754fd47c76-zdg46' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:21Z" name: error-404-isvc-ff919-predictor-754fd47c76.18d45122cbf48a90 namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 4db0fb9a-fadf-4606-a729-e462abfed1b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16512" uid: a1bddccf-af7a-4b39-9885-a3e58801ff0f kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Scaled up replica set error-404-isvc-ff919-predictor-754fd47c76 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:21Z" name: error-404-isvc-ff919-predictor.18d45122ca420429 namespace: kserve-ci-e2e-test resourceVersion: "16519" uid: bf7ccd9f-249a-477f-8973-0ff32c5815b6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: ab54ae6a-2127-4e33-a512-c3726878dbf0 kind: Event lastTimestamp: "2026-09-11T16:29:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:51Z" name: error-404-isvc-ff919-predictor.18d4512648ac1197 namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 69f96e27-7868-4b11-9584-26103c6a5ebc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16515" uid: ab54ae6a-2127-4e33-a512-c3726878dbf0 kind: Event lastTimestamp: "2026-09-11T16:29:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:51Z" name: error-404-isvc-ff919-predictor.18d45126490d8c09 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: f08269ab-f226-42c7-8688-e2a7458aff29 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:30:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: ab54ae6a-2127-4e33-a512-c3726878dbf0 kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: error-404-isvc-ff919-predictor.18d4512d4663b810 namespace: kserve-ci-e2e-test resourceVersion: "17291" uid: fb7ade16-a299-43ff-b8e8-78cb836d1bca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:30:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: ab54ae6a-2127-4e33-a512-c3726878dbf0 kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: error-404-isvc-ff919-predictor.18d4512d46ca33fd namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: ab62bb41-a0ed-46ca-8c77-aab011e71005 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 6c98b945-8d62-4adc-b607-87cff8e62592 kind: Event lastTimestamp: "2026-09-11T16:30:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:10Z" name: error-404-isvc-ff919.18d45122ca58caf2 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 5ace53da-d598-41a7-807b-162865195678 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 6c98b945-8d62-4adc-b607-87cff8e62592 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-ff919": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:21Z" name: error-404-isvc-ff919.18d45122cacf2297 namespace: kserve-ci-e2e-test resourceVersion: "16523" uid: d920e891-c0a3-4848-9920-b950c5a8940c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: 6c98b945-8d62-4adc-b607-87cff8e62592 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:21Z" name: error-404-isvc-ff919.18d45122cacf6f2c namespace: kserve-ci-e2e-test resourceVersion: "16526" uid: a9370031-a05d-4d5c-a394-88d25506768a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17054" uid: 6c98b945-8d62-4adc-b607-87cff8e62592 kind: Event lastTimestamp: "2026-09-11T16:30:10Z" message: InferenceService [error-404-isvc-ff919] is Ready metadata: creationTimestamp: "2026-09-11T16:30:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:10Z" name: error-404-isvc-ff919.18d4512e298139f3 namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: ca24f452-dc81-4971-b496-4277d1151b60 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:27:01.650510Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:27:01Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45102267e3d96 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: e330891a-5416-42ff-8de3-1a0cf02d1e5b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:01Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d4510234b6944e namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: 5b2a584a-1a70-46bd-b2dd-0365e2739fac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451026c54e8d0 namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 3e4102dd-6e59-438f-9488-cf9b2eada115 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4164f82a611b9e52a903d9686e415dbc9c35d0c556e392d7254ca59528ec9948" already present on machine metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451026e243404 namespace: kserve-ci-e2e-test resourceVersion: "14939" uid: 0742e983-9190-4cd7-8b43-41be9826f328 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451027189aeae namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 85bdcb4d-d974-4a52-9ced-4e36a3a0d38d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451027254c848 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: db5c5877-32ea-43fe-ac4a-72adc4679a34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" metadata: creationTimestamp: "2026-09-11T16:27:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:08Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45103a40de878 namespace: kserve-ci-e2e-test resourceVersion: "15009" uid: 87a5c9db-37fe-4742-88d3-1e43a1627218 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" in 10.606s (10.606s including waiting). Image size: 1583654804 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451061c47604f namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 0956a928-e6a0-456e-8e2f-3b01bd4a4755 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451061fdf2d51 namespace: kserve-ci-e2e-test resourceVersion: "15122" uid: e3506075-3cd2-4247-aeb4-c39a09faf60b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d4510620b53165 namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 9fca331d-53be-4db9-96df-bc3ed8d19265 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d4510620c4c1ea namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: a3ab2b05-f54f-4e45-bb34-902b5a8c978a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.551s (3.551s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45106f47bf5e8 namespace: kserve-ci-e2e-test resourceVersion: "15161" uid: 6ff7b5f9-b21d-4e54-8648-6f8b85378e28 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45106f9d1f11c namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: 16ef6eda-380d-420a-926e-462898e477ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45106fa70c9ba namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: c069f7e3-f03c-4a0e-9832-b93d5d739a2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-11T16:27:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:28:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:27:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:40Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d4510763224ac7 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 495ed4c3-236b-4b8d-9148-eb257c4ea15b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45122c39e13d9 namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: bad1a0f3-2aa7-4c52-a43f-abfe563498a1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d45122c3a0f097 namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: bd67b881-9034-4485-8f0a-b9d18ad9c5ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 6161918e-db69-4222-8340-14c7c05a98d5 kind: Event lastTimestamp: "2026-09-11T16:29:25Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:25Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf.18d451238f867f4a namespace: kserve-ci-e2e-test resourceVersion: "16602" uid: 9d48dd67-1e44-44b1-82c6-5edfb80e49db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5dcd4747bf namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: 690dd626-a5e7-46f6-b89a-9a055668a4d2 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5dcd4747bf-pxvpf' metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: isvc-sklearn-graph-1-predictor-5dcd4747bf.18d451022615d0d0 namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: 5f912e0b-1597-4d45-a4a8-82eed5a8be04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14774" uid: 1e298aed-c261-4978-9dd3-ea721cc33661 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5dcd4747bf from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: isvc-sklearn-graph-1-predictor.18d451021e1316e8 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: c340b443-abb3-4368-ae66-797130d46c13 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: ca0738e9-ba57-4c32-ba12-3b8e73a4e002 kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: isvc-sklearn-graph-1-predictor.18d45105a3ac040d namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: c6f87f62-6390-4149-bbee-33e2348ed65d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: ca0738e9-ba57-4c32-ba12-3b8e73a4e002 kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: isvc-sklearn-graph-1-predictor.18d45105a41ba5c2 namespace: kserve-ci-e2e-test resourceVersion: "15381" uid: ab635e64-9390-4c2a-bec3-a26dab553938 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: ca0738e9-ba57-4c32-ba12-3b8e73a4e002 kind: Event lastTimestamp: "2026-09-11T16:29:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:01Z" name: isvc-sklearn-graph-1-predictor.18d4511021c5e5e3 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: c6d7a885-83a2-400a-92c7-575a55b56834 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: ca0738e9-ba57-4c32-ba12-3b8e73a4e002 kind: Event lastTimestamp: "2026-09-11T16:29:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:01Z" name: isvc-sklearn-graph-1-predictor.18d4511022829756 namespace: kserve-ci-e2e-test resourceVersion: "16216" uid: 4fcac930-5e59-451a-9403-4ffb3ef61c79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: 409aca0b-8c4a-455a-940a-c7daacfdc381 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:46Z" name: isvc-sklearn-graph-1.18d45102244452db namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: d7d543f8-41d5-4173-97e0-b8437f81715b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 409aca0b-8c4a-455a-940a-c7daacfdc381 kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:46Z" name: isvc-sklearn-graph-1.18d4511aa3d14c74 namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 7a8345b9-e8ff-417e-a3e7-fe20a9e8648e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:27:02.069550Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451023f782e10 namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 73681dd5-70c2-4f52-aed2-780b2b3d0dfa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451024cddc500 namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: d7f6bf9e-f57a-4f9f-b38e-fb33eba6a140 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:27:03Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45102893aedb5 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: ca53f804-796c-4c49-a2cf-508ea9a27d95 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4164f82a611b9e52a903d9686e415dbc9c35d0c556e392d7254ca59528ec9948" already present on machine metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451028afe0f2c namespace: kserve-ci-e2e-test resourceVersion: "14956" uid: 62be34c3-f613-428a-9f7e-c93107c11a6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45102900123fa namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: f293226c-1b72-469f-961d-f9ec7b1b817c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451029195b20e namespace: kserve-ci-e2e-test resourceVersion: "14959" uid: 648e15c1-e46a-466d-a1b2-23c34a1964a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:08Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" metadata: creationTimestamp: "2026-09-11T16:27:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:08Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45103a3f05085 namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 800dd137-db13-439f-aed9-fbe7d36aba74 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1928" in 10.625s (10.625s including waiting). Image size: 1583654804 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451061d48eda2 namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 6dcafaa2-f614-492b-b34f-2728876b1255 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:20Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451069711aef5 namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: bf7b79c9-1ee8-4cf6-85bc-84d7a9b0bcf1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:20Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45106984be375 namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: bd52c603-0aa3-47a5-b4dd-a0df793be839 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:20Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-11T16:27:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:20Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d4510698627b0d namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 7025bc94-6afd-4113-9812-94c51539537a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.57s (1.57s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45106f5f9c4b2 namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 27ebf01d-64d3-417a-99eb-c8345d3d3f99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45106fa5ff3c5 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 84104f5a-6ab9-4ccc-a467-8ac0a64529df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45106fb068f97 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 8928f31a-eb08-4e56-8b51-46a2bae4be06 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-11T16:27:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:28:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:27:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:39Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d451072723f890 namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: dc232626-910b-476e-a9f4-80ee5ccd796e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45122c9fceff4 namespace: kserve-ci-e2e-test resourceVersion: "16517" uid: 3270b0e1-b8de-473a-98bd-e74e59d75776 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d45122c9fd4ccb namespace: kserve-ci-e2e-test resourceVersion: "16521" uid: fb89e914-e621-4292-8a39-5602412671b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 3ca7c6fd-d65b-4007-a311-8ea69cc6593a kind: Event lastTimestamp: "2026-09-11T16:29:24Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:24Z" name: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg.18d4512353a45b44 namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: f68ab9cb-3d05-40d4-9905-5d378633b35c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-76586f68dc namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: c0049264-20fc-46f0-af5a-f43b008bbc95 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-76586f68dc-g5gwg' metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-2-predictor-76586f68dc.18d451023ee6a90d namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 5e8a1f14-8432-4f90-9a89-e83afa4ea51e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: 4fa19892-664f-489c-95b3-e144ab498dc7 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-76586f68dc from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:02Z" name: isvc-sklearn-graph-2-predictor.18d451023c113c22 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 17a016d6-cfbd-41a0-9721-547e64646987 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:27:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: da5d76e8-0070-4c1a-86f4-5d87a478d4b5 kind: Event lastTimestamp: "2026-09-11T16:27:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:32Z" name: isvc-sklearn-graph-2-predictor.18d45105ba84f397 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 52421988-17eb-4938-9f88-b80063f0b789 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:27:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: da5d76e8-0070-4c1a-86f4-5d87a478d4b5 kind: Event lastTimestamp: "2026-09-11T16:27:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:32Z" name: isvc-sklearn-graph-2-predictor.18d45105bb07abe8 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 63101076-7f21-4262-a571-5674cacf1b04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:27:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: da5d76e8-0070-4c1a-86f4-5d87a478d4b5 kind: Event lastTimestamp: "2026-09-11T16:29:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:17Z" name: isvc-sklearn-graph-2-predictor.18d4510cb896edec namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: f92606b6-1785-44a1-88e1-9dd18d56d483 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:27:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: da5d76e8-0070-4c1a-86f4-5d87a478d4b5 kind: Event lastTimestamp: "2026-09-11T16:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:17Z" name: isvc-sklearn-graph-2-predictor.18d4510cb9038324 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: f8910bb4-09a2-462a-b236-40944b6e6099 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 4f60336c-6147-4e19-a848-0a32a5dfae85 kind: Event lastTimestamp: "2026-09-11T16:28:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:49Z" name: isvc-sklearn-graph-2.18d451023c1423ae namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 4024da97-014c-4aaa-98bd-0c96ec4c506b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16114" uid: 4f60336c-6147-4e19-a848-0a32a5dfae85 kind: Event lastTimestamp: "2026-09-11T16:28:49Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-11T16:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:49Z" name: isvc-sklearn-graph-2.18d4511b56aaf31c namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 32e3fa63-b3e9-41ee-b638-36570a610101 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:27:01.963660Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-x7bp5 to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:27:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d451023928800a namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: cb4c211a-a1b3-4c48-a237-88cbe81bbfd2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510246d978c3 namespace: kserve-ci-e2e-test resourceVersion: "14911" uid: b909677b-43d9-4d68-8c29-b9272b8dbcd1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:27:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510280bba980 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: bf0a6cc9-ef0c-4092-b1e4-8d91e8fe6b3f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:4164f82a611b9e52a903d9686e415dbc9c35d0c556e392d7254ca59528ec9948" already present on machine metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45102826bf874 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: e1e45fca-ffd7-4224-b241-4335effec627 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45102881c0734 namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 40fdc6d2-1cad-4f30-ac34-5a8e90d0f2d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:03Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:03Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-11T16:27:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45102891dd64f namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 30a9703b-11de-4054-bc4b-e308d3cd9096 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:08Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-11T16:27:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45103a433fb52 namespace: kserve-ci-e2e-test resourceVersion: "15010" uid: 1591966d-4028-49df-aaa6-4f95dd4fda58 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 30.472s (30.472s including waiting). Image size: 1332343041 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510abc802d7c namespace: kserve-ci-e2e-test resourceVersion: "15309" uid: fc3fddfe-8c16-419e-85d6-f6008b1fb4ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510abf7c1a34 namespace: kserve-ci-e2e-test resourceVersion: "15310" uid: a726e963-e5c9-412e-817d-6cbefa014cba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510ac0093530 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 73402099-c39a-46c8-aa38-f0771ccde988 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510ac01448c0 namespace: kserve-ci-e2e-test resourceVersion: "15312" uid: 902886a5-04b4-4e75-ba27-69261d84eb00 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510ac2a2549a namespace: kserve-ci-e2e-test resourceVersion: "15313" uid: afcad3df-c994-458f-b6b1-51624167bc36 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:27:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:27:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510ac311fa08 namespace: kserve-ci-e2e-test resourceVersion: "15315" uid: 0346f1ac-eb80-4ae2-80ad-121f6ce07182 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-11T16:27:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:29:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:27:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d4510ae550e1f4 namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: 25ffd51c-c4b3-486b-a3ed-7bf0dbd53b19 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45122c64ca4b6 namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 054edf88-1426-4f2a-8669-4078f5935c57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d45122c64d1d85 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 5e21e714-8c08-43c5-8f83-0e311f477586 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5 namespace: kserve-ci-e2e-test resourceVersion: "14870" uid: 3fdf1240-102f-4895-82a8-2f9cc1acb8c5 kind: Event lastTimestamp: "2026-09-11T16:29:25Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:29:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-x7bp5.18d451239399852b namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: cc40ee7e-0726-493f-afad-5c1e1002c74d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: fc5c35e0-1951-471f-91bf-29683334c9b9 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-x7bp5' metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d4510238b4767e namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: fbe93b6b-35f1-4838-ab57-156966afd727 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 65ad4f55-4c2f-41a8-9b02-00e3cf67f3f9 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: isvc-xgboost-graph-predictor.18d45102360bf96f namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: 98c345f3-03cb-4d25-b154-855223d9d46d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: b63927a4-7b50-4818-ba88-94a93dba024d kind: Event lastTimestamp: "2026-09-11T16:28:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:01Z" name: isvc-xgboost-graph-predictor.18d45105b46e019f namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: e50ec917-37c0-4b50-a9aa-367a74be8a44 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: b63927a4-7b50-4818-ba88-94a93dba024d kind: Event lastTimestamp: "2026-09-11T16:28:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:01Z" name: isvc-xgboost-graph-predictor.18d45105b4d73a81 namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 6055fbf0-ec49-4276-808a-a94d5b0bec58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:28:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: b63927a4-7b50-4818-ba88-94a93dba024d kind: Event lastTimestamp: "2026-09-11T16:29:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:17Z" name: isvc-xgboost-graph-predictor.18d45113af82b143 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: a2cd71f7-bed6-47f8-9b2e-4f0a63798c09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:28:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: b63927a4-7b50-4818-ba88-94a93dba024d kind: Event lastTimestamp: "2026-09-11T16:29:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:17Z" name: isvc-xgboost-graph-predictor.18d45113afe597f0 namespace: kserve-ci-e2e-test resourceVersion: "16376" uid: ec4b82d0-bc35-4aaf-b06d-d0c3e71375f9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 4abdd050-9ce9-41d3-b26b-82b653e6d609 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:45Z" name: isvc-xgboost-graph.18d45102360d15df namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 396af201-f9fe-4c58-8722-20e722773749 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: 4abdd050-9ce9-41d3-b26b-82b653e6d609 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:45Z" name: isvc-xgboost-graph.18d4511a497b56ae namespace: kserve-ci-e2e-test resourceVersion: "15912" uid: 6867aac8-1d77-42a9-bb45-d11237affa03 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:29:11.500260Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-6f97b8c58f-jd8dx to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:29:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:29:11Z" name: model-chainer-6f97b8c58f-jd8dx.18d4512062241e39 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: ebc76043-7ddb-49e7-b7fc-afa85d59029a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:11Z" name: model-chainer-6f97b8c58f-jd8dx.18d451206dd60ed2 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 14207236-a400-4062-8545-6dab81784e13 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:12Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:12Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:29:12Z" name: model-chainer-6f97b8c58f-jd8dx.18d45120a80a9d35 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 8da69a2c-9654-4294-9d2a-49435c19fe42 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:12Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:12Z" name: model-chainer-6f97b8c58f-jd8dx.18d45120a9327105 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: e49f4327-a5be-4c06-b9e7-de05c45261db reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:12Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-11T16:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:12Z" name: model-chainer-6f97b8c58f-jd8dx.18d45120ac2e147e namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: 3ae7c499-17f4-4339-8f77-5b991b2ad7da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:12Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-11T16:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:12Z" name: model-chainer-6f97b8c58f-jd8dx.18d45120aca6e399 namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 186c1f54-be3c-44b1-85e7-28797338d9c3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:21Z" name: model-chainer-6f97b8c58f-jd8dx.18d45122bc44bdb3 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: f72d7388-87d3-4588-9122-6e56158e45f7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-6f97b8c58f-jd8dx namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 80d98814-f0dd-447f-9957-0410f03b41c0 kind: Event lastTimestamp: "2026-09-11T16:29:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:48Z" name: model-chainer-6f97b8c58f-jd8dx.18d45123425e7462 namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: b3c2e027-cae4-438b-9a80-e160569df9d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-6f97b8c58f namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: e010e117-5a6f-45bc-bfdd-c4bfdff06d78 kind: Event lastTimestamp: "2026-09-11T16:29:11Z" message: 'Created pod: model-chainer-6f97b8c58f-jd8dx' metadata: creationTimestamp: "2026-09-11T16:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:11Z" name: model-chainer-6f97b8c58f.18d4512061bf7d45 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 0e58fc94-1ac6-4f43-adc0-b1f55f2a2aa4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 7eb9d067-7ed1-42a3-9350-a350b112b781 kind: Event lastTimestamp: "2026-09-11T16:29:11Z" message: Scaled up replica set model-chainer-6f97b8c58f from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:11Z" name: model-chainer.18d4512060f5041b namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: eedf7d71-9307-4a78-ad6b-cdcc325e6790 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:11Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: be141be0-bae1-4d1e-bf76-0892b8859d3e kind: Event lastTimestamp: "2026-09-11T16:29:11Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-11T16:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:11Z" name: model-chainer.18d4512064497c0f namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 6b65b52c-8d26-46a8-a1c9-dc7af48d4474 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:36.701612Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e5408-66fb7f646b-d99b8 to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:36Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b1f30bb6ef namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: 9e3affa6-6c54-4ed0-a51a-f881427cf46b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-e5408-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:36Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b1fd6f0ae2 namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: 0a1ddcd7-2190-467a-972d-6b859a8de6b7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:37Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:37Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:37Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b238bfc3ff namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 90b73771-b53a-4017-9a39-a3e56618dec2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e5408} kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:37Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:37Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b23a2f70c6 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: a307c8d4-0e39-4228-a426-75e7bb2c66c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e5408} kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:37Z" message: 'Created container: sequence-graph-e5408' metadata: creationTimestamp: "2026-09-11T16:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:37Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b23d4f12dd namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: 8e553359-abda-40c2-9456-0b4290364c3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e5408} kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:37Z" message: Started container sequence-graph-e5408 metadata: creationTimestamp: "2026-09-11T16:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:37Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b23dc3c9bc namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: d05fbe56-13bb-4151-b1e6-a19f1bc7c47f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e5408} kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Stopping container sequence-graph-e5408 metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:46Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b44b8467d4 namespace: kserve-ci-e2e-test resourceVersion: "21713" uid: 36dfbfe0-aae1-416c-9bb9-d59ce67e11a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e5408} kind: Pod name: sequence-graph-e5408-66fb7f646b-d99b8 namespace: kserve-ci-e2e-test resourceVersion: "21583" uid: 6c2e8e73-fc91-46ca-b629-ea76fec6d58a kind: Event lastTimestamp: "2026-09-11T16:40:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:14Z" name: sequence-graph-e5408-66fb7f646b-d99b8.18d451b4dcc2dddd namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: e60ceb27-ae2f-4953-99e6-7cf7c28d837f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e5408-66fb7f646b namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: 4173086b-7907-42bd-aeeb-c20794424c2f kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'Created pod: sequence-graph-e5408-66fb7f646b-d99b8' metadata: creationTimestamp: "2026-09-11T16:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: sequence-graph-e5408-66fb7f646b.18d451b1f29eb64c namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 373821a1-7aea-4ec3-9ffa-46f359c257d4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e5408 namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: 74a7b52b-6229-4aaf-b9ec-96584ba8f94b kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: Scaled up replica set sequence-graph-e5408-66fb7f646b from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: sequence-graph-e5408.18d451b1f1ca5ffe namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: f497f3d6-2d22-4bec-a254-07909c1ab2a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e5408 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 8dd2aa00-286c-41d5-a7c2-b42b84e945f8 kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: InferenceGraph [sequence-graph-e5408] is Ready metadata: creationTimestamp: "2026-09-11T16:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:36Z" name: sequence-graph-e5408.18d451b1f76e5074 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: b38c8969-b1e3-4ba4-ade5-9762472eb694 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:40:46.998009Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22400" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ea3c5-5d669fffd4-pqsgm to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:40:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:40:46Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c251098ae6 namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: 48c4f789-9a07-47b6-8fcc-533320cbf62b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:47Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:40:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-ea3c5-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:47Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c25f5195f9 namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: a19f2330-4e22-45d7-a985-d3311aa12b0f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22411" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:40:48Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:40:48Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c296adfa6c namespace: kserve-ci-e2e-test resourceVersion: "22429" uid: 28b0a1c7-2dfa-42bb-a3ab-9a947e5659cb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ea3c5} kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:40:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:48Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c297dd7652 namespace: kserve-ci-e2e-test resourceVersion: "22431" uid: 0747ecc2-f107-4016-aa54-1027c4694ae5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ea3c5} kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:40:48Z" message: 'Created container: sequence-graph-ea3c5' metadata: creationTimestamp: "2026-09-11T16:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:48Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c29ad3dc80 namespace: kserve-ci-e2e-test resourceVersion: "22432" uid: 62101c8f-a9e0-4843-a018-e70892cc80ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ea3c5} kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:40:48Z" message: Started container sequence-graph-ea3c5 metadata: creationTimestamp: "2026-09-11T16:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:40:48Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d451c29b4d5660 namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: e99839a0-f164-4fe0-a549-6aa28c879f09 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ea3c5} kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Stopping container sequence-graph-ea3c5 metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:01Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d4523579cdf94a namespace: kserve-ci-e2e-test resourceVersion: "26296" uid: 740f7552-a8b9-486b-ae6d-91d5376e70cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ea3c5} kind: Pod name: sequence-graph-ea3c5-5d669fffd4-pqsgm namespace: kserve-ci-e2e-test resourceVersion: "22403" uid: bbaf5cf1-a97b-40f5-b77c-0c8365aa5e3a kind: Event lastTimestamp: "2026-09-11T16:49:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:29Z" name: sequence-graph-ea3c5-5d669fffd4-pqsgm.18d4523624df7214 namespace: kserve-ci-e2e-test resourceVersion: "26704" uid: 07ed6623-d0e4-4d00-8122-7f6f9c20451c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ea3c5-5d669fffd4 namespace: kserve-ci-e2e-test resourceVersion: "22397" uid: 88953438-71ff-4e28-8b2a-b5f9e3d922df kind: Event lastTimestamp: "2026-09-11T16:40:46Z" message: 'Created pod: sequence-graph-ea3c5-5d669fffd4-pqsgm' metadata: creationTimestamp: "2026-09-11T16:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:46Z" name: sequence-graph-ea3c5-5d669fffd4.18d451c2506a5a1c namespace: kserve-ci-e2e-test resourceVersion: "22402" uid: 295affa7-c750-48a3-b9a7-3cc4d4785976 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22395" uid: 34c70100-04a1-46ab-a6ea-a88a06fa7518 kind: Event lastTimestamp: "2026-09-11T16:40:46Z" message: Scaled up replica set sequence-graph-ea3c5-5d669fffd4 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:40:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:46Z" name: sequence-graph-ea3c5.18d451c24f6737d8 namespace: kserve-ci-e2e-test resourceVersion: "22398" uid: 20885e08-2ec3-4ad3-8754-db27df8ba359 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: c410361b-8e5a-47fe-b48b-81a6cd61d99f kind: Event lastTimestamp: "2026-09-11T16:40:47Z" message: InferenceGraph [sequence-graph-ea3c5] is Ready metadata: creationTimestamp: "2026-09-11T16:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:47Z" name: sequence-graph-ea3c5.18d451c2552d6d19 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: 9ae8b059-2e96-46d7-8605-2afdb5b51975 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:41:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22396" uid: 498a17ac-827c-40c7-87e5-24974d47274f kind: Event lastTimestamp: "2026-09-11T16:41:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:41:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:41:16Z" name: sequence-graph-ea3c5.18d451c5cdd79b41 namespace: kserve-ci-e2e-test resourceVersion: "22648" uid: 4284419f-2195-4236-aa09-3aa3c48d0943 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:41:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22396" uid: 498a17ac-827c-40c7-87e5-24974d47274f kind: Event lastTimestamp: "2026-09-11T16:41:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:41:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:41:16Z" name: sequence-graph-ea3c5.18d451c5ce1940ee namespace: kserve-ci-e2e-test resourceVersion: "22649" uid: 72739007-448c-4090-9726-12a2941e98ca reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:30:21.801902Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17146" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ff919-c4589574f-j5k57 to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919-c4589574f-j5k57.18d45130c072157a namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: f09e91f5-4c6d-40ec-9bde-83f60f469f9d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:22Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:30:22Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:30:22Z" name: sequence-graph-ff919-c4589574f-j5k57.18d45130e265724d namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: 7ce285e2-2636-469a-96e3-f3aba66ab48c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ff919} kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:30:22Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:30:22Z" name: sequence-graph-ff919-c4589574f-j5k57.18d45130e394558e namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: 7915d1fb-fb36-45b2-ac93-0fe0a61616eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ff919} kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:30:22Z" message: 'Created container: sequence-graph-ff919' metadata: creationTimestamp: "2026-09-11T16:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:30:22Z" name: sequence-graph-ff919-c4589574f-j5k57.18d45130e696b798 namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 22ca625e-10f2-4d5f-bcc2-66c2ecf46d1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ff919} kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:30:22Z" message: Started container sequence-graph-ff919 metadata: creationTimestamp: "2026-09-11T16:30:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:30:22Z" name: sequence-graph-ff919-c4589574f-j5k57.18d45130e70f5140 namespace: kserve-ci-e2e-test resourceVersion: "17176" uid: e63981a3-f560-4e7c-b18d-09671bfcad50 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ff919} kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Stopping container sequence-graph-ff919 metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: sequence-graph-ff919-c4589574f-j5k57.18d451a3ef2c95f1 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 61a32cb1-3295-4631-a516-4715f3d7dc2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ff919} kind: Pod name: sequence-graph-ff919-c4589574f-j5k57 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 63cf65da-9c6e-49cd-88b6-2f8b973ec491 kind: Event lastTimestamp: "2026-09-11T16:39:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:38:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:04Z" name: sequence-graph-ff919-c4589574f-j5k57.18d451a48bbe6dac namespace: kserve-ci-e2e-test resourceVersion: "21142" uid: 29079d33-22e6-4ef4-bbd2-2bf4c950a641 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ff919-c4589574f namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 6acbaf70-5a93-4904-a1a0-970d25e7870a kind: Event lastTimestamp: "2026-09-11T16:30:21Z" message: 'Created pod: sequence-graph-ff919-c4589574f-j5k57' metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919-c4589574f.18d45130bfd801fb namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: aa551221-0947-48c7-bb32-67c0c478d572 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 0fa563a0-71f9-48d1-9e76-706ae1d34a5d kind: Event lastTimestamp: "2026-09-11T16:30:21Z" message: Scaled up replica set sequence-graph-ff919-c4589574f from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919.18d45130beeebc29 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 4451cad8-8bc2-4fb7-8454-12221c5b4c26 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 19f651aa-4386-4dc1-8dba-0c2058cc3b3b kind: Event lastTimestamp: "2026-09-11T16:30:21Z" message: InferenceGraph [sequence-graph-ff919] is Ready metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919.18d45130c46e6698 namespace: kserve-ci-e2e-test resourceVersion: "17161" uid: 7de912bf-a861-452e-bf32-efb69fec37a1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 19f651aa-4386-4dc1-8dba-0c2058cc3b3b kind: Event lastTimestamp: "2026-09-11T16:30:21Z" message: 'Failed to update status for InferenceGraph "sequence-graph-ff919": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919.18d45130c6c1e011 namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 7ead2a8c-33f7-44fc-ae58-b655d4abc848 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: 19f651aa-4386-4dc1-8dba-0c2058cc3b3b kind: Event lastTimestamp: "2026-09-11T16:30:21Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:30:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:21Z" name: sequence-graph-ff919.18d45130c6c22a87 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: b19e21e7-be89-48d7-a834-32854f24f521 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 970ff472-6e3b-4c53-b2cf-0d219a867bc8 kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: sequence-graph-ff919.18d451343d603af4 namespace: kserve-ci-e2e-test resourceVersion: "17284" uid: 8b5b4c05-b4e5-493b-b31c-53d25bbdcc52 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 970ff472-6e3b-4c53-b2cf-0d219a867bc8 kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: sequence-graph-ff919.18d451343dc92764 namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: ded2b981-b6a1-44f1-979c-ba56a193c2f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 970ff472-6e3b-4c53-b2cf-0d219a867bc8 kind: Event lastTimestamp: "2026-09-11T16:30:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:51Z" name: sequence-graph-ff919.18d45137bc852de0 namespace: kserve-ci-e2e-test resourceVersion: "17394" uid: 666b49da-29f9-42cc-8888-50f396b38fb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17285" uid: 970ff472-6e3b-4c53-b2cf-0d219a867bc8 kind: Event lastTimestamp: "2026-09-11T16:30:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:51Z" name: sequence-graph-ff919.18d45137bcf719ec namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: 8944b985-d985-4ef5-8289-b1bbd01f3bd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:50:36.410494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27457" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-20f8f-7765c6fc54-q5zts to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:50:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:50:36Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d4524b8cc251eb namespace: kserve-ci-e2e-test resourceVersion: "27463" uid: a221947c-7cd0-4cb4-88a8-dc3bf402eec9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:50:36Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:50:36Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d4524baea9190b namespace: kserve-ci-e2e-test resourceVersion: "27482" uid: 9395cb34-e809-4a7b-8f03-d6a5c4e2b0a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-20f8f} kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:50:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:37Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d4524bafce0d71 namespace: kserve-ci-e2e-test resourceVersion: "27484" uid: 90919624-3730-4b39-9734-61a17b0c93b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-20f8f} kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:50:37Z" message: 'Created container: splitter-graph-20f8f' metadata: creationTimestamp: "2026-09-11T16:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:37Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d4524bb2a888bb namespace: kserve-ci-e2e-test resourceVersion: "27486" uid: bc26a5f2-c9e9-4200-b517-7e932b36f177 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-20f8f} kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:50:37Z" message: Started container splitter-graph-20f8f metadata: creationTimestamp: "2026-09-11T16:50:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:37Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d4524bb3211285 namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: 7334c13c-91c1-4a99-9cde-a02527eb52d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-20f8f} kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:58:50Z" message: Stopping container splitter-graph-20f8f metadata: creationTimestamp: "2026-09-11T16:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:58:50Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d452beb40471a8 namespace: kserve-ci-e2e-test resourceVersion: "30922" uid: 8fabe46f-4c98-4c13-888e-f40ecb3b53a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:58:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-20f8f} kind: Pod name: splitter-graph-20f8f-7765c6fc54-q5zts namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 9d40ee3a-3454-445e-bc1d-3ed7f572adc1 kind: Event lastTimestamp: "2026-09-11T16:59:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:59:18Z" name: splitter-graph-20f8f-7765c6fc54-q5zts.18d452bf535d7454 namespace: kserve-ci-e2e-test resourceVersion: "31159" uid: de8150e5-e4e9-4d6d-8ef9-406ad9ffc9b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-20f8f-7765c6fc54 namespace: kserve-ci-e2e-test resourceVersion: "27454" uid: 6016a5d0-84b2-4867-a3a0-319eee105b5e kind: Event lastTimestamp: "2026-09-11T16:50:36Z" message: 'Created pod: splitter-graph-20f8f-7765c6fc54-q5zts' metadata: creationTimestamp: "2026-09-11T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:36Z" name: splitter-graph-20f8f-7765c6fc54.18d4524b8c4fc908 namespace: kserve-ci-e2e-test resourceVersion: "27459" uid: 0dc99e0c-4f26-4886-a203-d265929924e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27452" uid: c1aee1ad-e16c-4e43-bd32-cff108f3a08a kind: Event lastTimestamp: "2026-09-11T16:50:36Z" message: Scaled up replica set splitter-graph-20f8f-7765c6fc54 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:36Z" name: splitter-graph-20f8f.18d4524b8b6d39aa namespace: kserve-ci-e2e-test resourceVersion: "27455" uid: 3dadfdbc-3a9a-4a0a-8a81-83e2e785454b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27470" uid: 85302165-a570-4ce2-9f8f-4729e9a05bd9 kind: Event lastTimestamp: "2026-09-11T16:50:36Z" message: InferenceGraph [splitter-graph-20f8f] is Ready metadata: creationTimestamp: "2026-09-11T16:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:36Z" name: splitter-graph-20f8f.18d4524b90627f12 namespace: kserve-ci-e2e-test resourceVersion: "27472" uid: 30cc5278-d3fd-4a2b-a8c8-73a40fc0af3f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: cb458b9e-66f4-4238-b5be-3105bf5c438f kind: Event lastTimestamp: "2026-09-11T16:51:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:51:06Z" name: splitter-graph-20f8f.18d4524f09d9413e namespace: kserve-ci-e2e-test resourceVersion: "27704" uid: bd443c7b-81cb-433c-8145-f60c83319731 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:50:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27453" uid: cb458b9e-66f4-4238-b5be-3105bf5c438f kind: Event lastTimestamp: "2026-09-11T16:51:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:51:06Z" name: splitter-graph-20f8f.18d4524f0a156f13 namespace: kserve-ci-e2e-test resourceVersion: "27705" uid: a94b5826-5b9a-4633-b551-3e17e9fc9cb6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:49:26.111594Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-ddc34-98b445d6c-9zb8b to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523b2e9e5005 namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 2191debf-70fe-42fd-a977-6a0928de68d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26658" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523b5086a95b namespace: kserve-ci-e2e-test resourceVersion: "26676" uid: e63f41a7-7dda-4c48-b03a-b244f6d8c267 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ddc34} kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523b51bdbe36 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: 1746d7f2-e4ea-4b62-b389-549816b58e34 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ddc34} kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'Created container: splitter-graph-ddc34' metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523b5491075e namespace: kserve-ci-e2e-test resourceVersion: "26679" uid: e046712b-2477-4a9d-bf68-4ac14d2776d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ddc34} kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: Started container splitter-graph-ddc34 metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523b554de0bf namespace: kserve-ci-e2e-test resourceVersion: "26680" uid: e4aec4aa-f6b9-49d4-b469-3084cac816f3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ddc34} kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Stopping container splitter-graph-ddc34 metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:36Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523d870c0c82 namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: 03836f3b-053f-4836-a986-f5f1a8557745 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-ddc34} kind: Pod name: splitter-graph-ddc34-98b445d6c-9zb8b namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 109e6e3d-44c2-4115-9b7d-4bb6fc9ffb5a kind: Event lastTimestamp: "2026-09-11T16:50:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:03Z" name: splitter-graph-ddc34-98b445d6c-9zb8b.18d4523e0b320ddb namespace: kserve-ci-e2e-test resourceVersion: "27167" uid: 951750c9-f475-4e55-b236-43483166b823 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-ddc34-98b445d6c namespace: kserve-ci-e2e-test resourceVersion: "26643" uid: 373012dc-1429-49a5-b059-6192a19c446e kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'Created pod: splitter-graph-ddc34-98b445d6c-9zb8b' metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34-98b445d6c.18d4523b2e132f3b namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 82764113-b2d6-4a76-b84f-deb1ac7a6757 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 5f6a6df9-7dbe-4f07-949d-fdea68e516f1 kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: Scaled up replica set splitter-graph-ddc34-98b445d6c from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34.18d4523b2d4e1942 namespace: kserve-ci-e2e-test resourceVersion: "26646" uid: e902be92-487e-4101-9af0-8a09bbe65040 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "26663" uid: 5210db64-46c9-4685-83d8-7d50ad831322 kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: InferenceGraph [splitter-graph-ddc34] is Ready metadata: creationTimestamp: "2026-09-11T16:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:26Z" name: splitter-graph-ddc34.18d4523b32f6f72d namespace: kserve-ci-e2e-test resourceVersion: "26665" uid: 774f632d-a171-434c-b501-d69fbe352134 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:11.124569Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1ab9e-predictor-595c484fbc-shn4m to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451abfe88fb9d namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 2442d505-67ec-485b-9d85-32da1d6fadce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-1ab9e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac09c0c7bd namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: 5350fdcc-382c-40c0-a349-d3ab5b1487ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac441639b4 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: b34950f6-f4f8-4155-a3ba-2ac4487ee8a8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac45607062 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: 17c43592-70b7-40bb-b92a-5e21a4c4bf0f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac484a14b7 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 9eef86e7-1016-44a1-ba29-5176c1b70b43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac490ce18e namespace: kserve-ci-e2e-test resourceVersion: "21332" uid: 3c74263c-d322-4694-8014-7235a6067541 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac4916e7c8 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 0d1112e8-a24f-478b-a0c9-b3d0deaeaeb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac4bc45cb7 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 94e8d361-aaa7-4ee8-8033-b1a43e6b0f86 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:12Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac4c372834 namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: 925a9098-6cab-4a24-b5ff-8bc81836eba9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:39:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:49Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d451ac9fb7d29f namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: cc24628a-c7bd-42d7-b491-15f73c271f49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:48:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:25Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d4522d2ed9f77c namespace: kserve-ci-e2e-test resourceVersion: "25881" uid: 238f5ec5-6467-479f-bde7-b6a3e3c0ba65 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m namespace: kserve-ci-e2e-test resourceVersion: "21277" uid: 96ef3781-b318-49f5-a13d-f7cc49f05109 kind: Event lastTimestamp: "2026-09-11T16:48:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:25Z" name: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m.18d4522d2edab5bb namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 10fdd967-c790-40b7-ba6c-dc06351bd4c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1ab9e-predictor-595c484fbc namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 5650a0f3-51e3-4553-95dc-9aee1fbc2cd5 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'Created pod: success-200-isvc-1ab9e-predictor-595c484fbc-shn4m' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e-predictor-595c484fbc.18d451abfe210390 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: dd96997f-d008-4dd7-b0fc-21c94afb5564 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21260" uid: aef9909f-37ae-427e-8bfa-49bc6b244980 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Scaled up replica set success-200-isvc-1ab9e-predictor-595c484fbc from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e-predictor.18d451abfc47eff7 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: a69e0041-54c6-444d-9f78-3f42a72df8c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: ede7a343-ea28-48c1-8dea-0f50896a69d7 kind: Event lastTimestamp: "2026-09-11T16:39:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:56Z" name: success-200-isvc-1ab9e-predictor.18d451af7b233f76 namespace: kserve-ci-e2e-test resourceVersion: "21916" uid: 526965c2-7a8d-4cfb-a395-110eef12a29b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:39:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: ede7a343-ea28-48c1-8dea-0f50896a69d7 kind: Event lastTimestamp: "2026-09-11T16:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:39:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:56Z" name: success-200-isvc-1ab9e-predictor.18d451af7b88ec95 namespace: kserve-ci-e2e-test resourceVersion: "21918" uid: 1775b8c5-ebca-4711-9d1d-5efa52983a13 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: ede7a343-ea28-48c1-8dea-0f50896a69d7 kind: Event lastTimestamp: "2026-09-11T16:40:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:41Z" name: success-200-isvc-1ab9e-predictor.18d451b2fa603833 namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 735abd40-3daf-4d04-aff2-ac92b28d6422 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:39:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1ab9e-predictor namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: ede7a343-ea28-48c1-8dea-0f50896a69d7 kind: Event lastTimestamp: "2026-09-11T16:40:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:41Z" name: success-200-isvc-1ab9e-predictor.18d451b2fabf9bc9 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: e6ee0fd4-9ca5-4a0e-80b1-c5f8f7a2a7ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: daa65f99-912f-4e33-8568-c54312d024ae kind: Event lastTimestamp: "2026-09-11T16:39:59Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:59Z" name: success-200-isvc-1ab9e.18d451abfc4e32bb namespace: kserve-ci-e2e-test resourceVersion: "21962" uid: 44aca9f0-6b0e-4425-b921-8e9143f83ee6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: daa65f99-912f-4e33-8568-c54312d024ae kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-1ab9e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1ab9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e.18d451abfcf00747 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: c7a23c4d-7140-489e-86c0-50e8d3150eb6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: daa65f99-912f-4e33-8568-c54312d024ae kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-1ab9e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-1ab9e.18d451abfcf06a93 namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 7dbd4d56-cd98-4583-9b9e-5f964d7ebd79 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1ab9e namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: daa65f99-912f-4e33-8568-c54312d024ae kind: Event lastTimestamp: "2026-09-11T16:39:59Z" message: InferenceService [success-200-isvc-1ab9e] is Ready metadata: creationTimestamp: "2026-09-11T16:39:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:59Z" name: success-200-isvc-1ab9e.18d451b75b4a7a1d namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: 18ef1aad-5e65-49d6-8093-91d01dc520db reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:49:36.406757Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523d9442045b namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 5951e973-05b7-4319-9327-c155558050fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26838" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523db6a99c36 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 4547a7db-1101-483a-a0d0-4466545c7e4d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523db815a62a namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: d54e634d-7622-4a57-bfd5-1aceaab6770b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523dbb14f5b6 namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: 9c911457-babb-4ca6-b648-c9ab2424c135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523dbb965852 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: baf1ee2c-e81d-4b1a-bbd1-9a3521598d41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523dbba4276b namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 65ffb627-d461-4bd7-a12c-2bb25b6ed35e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523dbebb68c5 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: b46b5337-70bf-4ac4-8819-8072b441a267 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:49:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:37Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523dbf40e9c4 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 2765a39e-aa52-493c-b6f5-fb4a34476864 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:50:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:15Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d4523e3278fcc4 namespace: kserve-ci-e2e-test resourceVersion: "27268" uid: 469ae6a3-7576-4a55-8e84-2ad9fe70cf20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:58:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:58:51Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d452bebbaded5e namespace: kserve-ci-e2e-test resourceVersion: "30940" uid: 8052eadb-4d65-48f0-8b35-d3be3070eabc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 075be758-ad85-458e-bf71-934e5a2f06a6 kind: Event lastTimestamp: "2026-09-11T16:58:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:58:51Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4.18d452bebbae6b44 namespace: kserve-ci-e2e-test resourceVersion: "30946" uid: 7bdc8cba-9103-4a29-aec9-94fc55c17956 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-20f8f-predictor-7996fdcc9f namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: 599acbc8-fa1b-40ff-a3e6-83853ae88bdd kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: 'Created pod: success-200-isvc-20f8f-predictor-7996fdcc9f-82nm4' metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-20f8f-predictor-7996fdcc9f.18d4523d93d6c93f namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 7f519e45-36d8-48b1-afd0-169ce88af5b2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: eb7be6f5-2a60-464c-847e-ebd31911709d kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Scaled up replica set success-200-isvc-20f8f-predictor-7996fdcc9f from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-20f8f-predictor.18d4523d921461bf namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 3661463e-88ba-4420-9f4f-53f853d91237 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 409ce6f8-6a62-4939-8d9e-6b83d7367252 kind: Event lastTimestamp: "2026-09-11T16:50:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:06Z" name: success-200-isvc-20f8f-predictor.18d4524110b5e30e namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: ca9cdaab-46b1-4344-a916-364ffa32c079 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:49:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 409ce6f8-6a62-4939-8d9e-6b83d7367252 kind: Event lastTimestamp: "2026-09-11T16:50:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:06Z" name: success-200-isvc-20f8f-predictor.18d4524111181782 namespace: kserve-ci-e2e-test resourceVersion: "27196" uid: d863ce06-2b0f-465a-b2fc-5d8cde19ccf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 409ce6f8-6a62-4939-8d9e-6b83d7367252 kind: Event lastTimestamp: "2026-09-11T16:50:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:51Z" name: success-200-isvc-20f8f-predictor.18d452480e85c722 namespace: kserve-ci-e2e-test resourceVersion: "27595" uid: 4ad94507-d5b6-4771-9a29-ad4871be429d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:50:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-20f8f-predictor namespace: kserve-ci-e2e-test resourceVersion: "27040" uid: 409ce6f8-6a62-4939-8d9e-6b83d7367252 kind: Event lastTimestamp: "2026-09-11T16:50:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:51Z" name: success-200-isvc-20f8f-predictor.18d452480eedde47 namespace: kserve-ci-e2e-test resourceVersion: "27596" uid: a2a4aefa-2234-4aa0-9534-236b6565b05e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20f8f namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: b86d795e-ab24-4f60-951b-b4f10f49ed80 kind: Event lastTimestamp: "2026-09-11T16:50:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:25Z" name: success-200-isvc-20f8f.18d4523d9219d49c namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: 3a209396-dee2-4b57-926a-a52d0a1e31db reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-20f8f namespace: kserve-ci-e2e-test resourceVersion: "27361" uid: b86d795e-ab24-4f60-951b-b4f10f49ed80 kind: Event lastTimestamp: "2026-09-11T16:50:25Z" message: InferenceService [success-200-isvc-20f8f] is Ready metadata: creationTimestamp: "2026-09-11T16:50:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:25Z" name: success-200-isvc-20f8f.18d45248ed9c4ce1 namespace: kserve-ci-e2e-test resourceVersion: "27362" uid: af649bac-2d9f-4373-befc-44a9a144fb29 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:49:01.835765Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26349" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-482a1-predictor-747c4d74f5-fq6rz to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:49:01Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d4523587aa7e42 namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: 2d7974ae-792f-47ea-91fd-cff5eb55d13d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235a9d795ca namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: 0f6f9a41-4aae-4cf4-961e-afb51bd5c8f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235ab172b5e namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: ac63a9ec-fd0f-4a0c-bd5e-a35fdc0174c5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235adc71c2a namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 2e0bc866-f742-4f38-955f-ecf28a16a6b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235ae6885e8 namespace: kserve-ci-e2e-test resourceVersion: "26403" uid: 528347ee-cc66-4860-879b-899b3d6dad43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235ae7553c7 namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: 1034681e-fa45-45fe-955d-1ebfb3de0d76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235b1516316 namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: e4559bad-f072-44dc-9c45-3fe75bb2fdfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:02Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235b1cc7ef8 namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: 8b37bd23-cab7-4ef0-adb1-510e3e2c621f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:49:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T16:49:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:38Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45235c874ac2d namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: bdaf6d9d-acd2-44fe-aa27-beed56f83697 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T17:06:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:21Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45327822bdecf namespace: kserve-ci-e2e-test resourceVersion: "34061" uid: e92d83c8-b750-44cb-a583-222d312be893 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T17:06:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:21Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d45327822c5320 namespace: kserve-ci-e2e-test resourceVersion: "34067" uid: 76d455c1-33a3-4748-a2bd-cb6454c6731a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: ac5018e8-bd51-4c7b-a22e-08d79473152e kind: Event lastTimestamp: "2026-09-11T17:06:23Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T17:06:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:23Z" name: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz.18d4532829169ab9 namespace: kserve-ci-e2e-test resourceVersion: "34138" uid: 116d79b4-92c1-4e2c-a2be-425b83a89807 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-482a1-predictor-747c4d74f5 namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: bc0e062f-c378-4051-8341-e6cc51d0d135 kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: 'Created pod: success-200-isvc-482a1-predictor-747c4d74f5-fq6rz' metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:01Z" name: success-200-isvc-482a1-predictor-747c4d74f5.18d452358738d4e4 namespace: kserve-ci-e2e-test resourceVersion: "26350" uid: 6fe77b75-93e8-403a-87f8-64bd61d7d70b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: c5e48804-a6e3-4afc-be1c-c4fd17fa9176 kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Scaled up replica set success-200-isvc-482a1-predictor-747c4d74f5 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:01Z" name: success-200-isvc-482a1-predictor.18d452358554ba37 namespace: kserve-ci-e2e-test resourceVersion: "26344" uid: ee2d49ca-b065-42cc-ae5c-675188b9ab3e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: a3b93742-c751-4a6f-993a-e190b17b9894 kind: Event lastTimestamp: "2026-09-11T16:49:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:31Z" name: success-200-isvc-482a1-predictor.18d45239041e17b0 namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 7496810f-4cdc-4338-9663-dfd3a518ffbe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: a3b93742-c751-4a6f-993a-e190b17b9894 kind: Event lastTimestamp: "2026-09-11T16:49:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:31Z" name: success-200-isvc-482a1-predictor.18d4523904f75e14 namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 7229fc9c-2484-43e6-90b5-2dae41bae6de reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: a3b93742-c751-4a6f-993a-e190b17b9894 kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: success-200-isvc-482a1-predictor.18d452400269a69b namespace: kserve-ci-e2e-test resourceVersion: "27282" uid: b3ccdfe8-d7fc-47c9-a48d-3dc4cf773be6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-482a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: a3b93742-c751-4a6f-993a-e190b17b9894 kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: success-200-isvc-482a1-predictor.18d4524002cab82a namespace: kserve-ci-e2e-test resourceVersion: "27283" uid: bb428584-4a8b-45e8-a0e7-54a9ea980d7f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-482a1 namespace: kserve-ci-e2e-test resourceVersion: "26329" uid: e3d83331-5cc5-4bd7-ba10-55fa5ad9434e kind: Event lastTimestamp: "2026-09-11T16:49:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:49Z" name: success-200-isvc-482a1.18d452358558ba35 namespace: kserve-ci-e2e-test resourceVersion: "27014" uid: 7a3e4178-647d-4661-ba4d-898fd0f48772 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27011" uid: e3d83331-5cc5-4bd7-ba10-55fa5ad9434e kind: Event lastTimestamp: "2026-09-11T16:49:49Z" message: InferenceService [success-200-isvc-482a1] is Ready metadata: creationTimestamp: "2026-09-11T16:49:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:49Z" name: success-200-isvc-482a1.18d45240835d8452 namespace: kserve-ci-e2e-test resourceVersion: "27012" uid: c9154e1e-3942-43fe-b6c3-abd6e7278b43 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:27:01.750270Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8975a-predictor-67685ddcd7-57fpp to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:27:01Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451022c70631c namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: dda98662-de57-468b-9cce-803ba3f580a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14823" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:27:02Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451024f132fac namespace: kserve-ci-e2e-test resourceVersion: "14920" uid: 2a77c49b-6e68-4e7b-a74f-9dba4ba7966f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:02Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" metadata: creationTimestamp: "2026-09-11T16:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:02Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d4510250cd9b39 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: deb5a7b1-9293-4b27-a729-00f39c6099ed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" in 16.231s (16.231s including waiting). Image size: 1356712016 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d4510618420130 namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: a079a37a-93a3-4672-959d-d29cd24a6698 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451061bfb8d88 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: e2c8895c-e73a-4684-b4b1-ec2d0f99556a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451061c8277f3 namespace: kserve-ci-e2e-test resourceVersion: "15119" uid: 66da84ec-f751-4abe-a13d-17557031b67a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-11T16:27:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:18Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451061c90c9c7 namespace: kserve-ci-e2e-test resourceVersion: "15120" uid: 1a16b47b-3365-49b8-9fd0-35e0b458aa23 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.643s (3.643s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d45106f5bca6c8 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: 622c4b63-1c86-4a44-88dd-220020ebc5a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d45106f9f143c6 namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: dca98626-b316-4279-9af6-83e07557eebc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:27:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:27:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:27:22Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d45106fab54518 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 8f6af7d0-b650-49a7-a2e1-96f32dd5c59c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-11T16:27:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:28:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:27:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:49Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d451072707af36 namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 75e28416-7995-48e0-a3db-abacfff82393 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d4511a69a291ca namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 25f4ffa0-e7ac-4df8-aaf5-e1c0bb69a605 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d4511a69a3ea1e namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: 8a056950-d06e-408d-9a50-fcd0b8468627 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 817b4430-0c4f-4d23-af34-12bf6e0fe825 kind: Event lastTimestamp: "2026-09-11T16:28:49Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:28:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:49Z" name: success-200-isvc-8975a-predictor-67685ddcd7-57fpp.18d4511b2d7a1e6a namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: 622478d3-9fd1-4d87-a4c8-b53e9b342ef5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8975a-predictor-67685ddcd7 namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 25140bb6-297f-44f3-9811-a1df03eff514 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: 'Created pod: success-200-isvc-8975a-predictor-67685ddcd7-57fpp' metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: success-200-isvc-8975a-predictor-67685ddcd7.18d451022c035a80 namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: cad43a8c-f6c7-472e-8556-9fd00875138f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 753cdd3d-8dd8-4ea8-a839-84d0dced8621 kind: Event lastTimestamp: "2026-09-11T16:27:01Z" message: Scaled up replica set success-200-isvc-8975a-predictor-67685ddcd7 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:01Z" name: success-200-isvc-8975a-predictor.18d451022a59b5e9 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 801f6d83-88c0-458d-a978-89c0a3aeb924 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: ac07b2a8-addc-45b0-96e1-99a20ca3e33c kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: success-200-isvc-8975a-predictor.18d45105a8ccedf0 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 798a0adc-e376-4718-ba96-342bba088f55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:27:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: ac07b2a8-addc-45b0-96e1-99a20ca3e33c kind: Event lastTimestamp: "2026-09-11T16:27:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:27:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:27:46Z" name: success-200-isvc-8975a-predictor.18d45105a935b4cb namespace: kserve-ci-e2e-test resourceVersion: "15383" uid: 4d8a8ba5-35e4-47ef-8e0b-97f63475fb17 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: ac07b2a8-addc-45b0-96e1-99a20ca3e33c kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: success-200-isvc-8975a-predictor.18d451102573b36b namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: 137ccb51-c87f-4c00-8716-b77adc1f0912 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:28:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8975a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: ac07b2a8-addc-45b0-96e1-99a20ca3e33c kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:28:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: success-200-isvc-8975a-predictor.18d4511025da1518 namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: fa1234c9-1852-4bef-aa3d-a3bfcca82b0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:27:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8975a namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: e562be37-23f7-45ee-a688-6dbea682ae0d kind: Event lastTimestamp: "2026-09-11T16:28:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:27:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:19Z" name: success-200-isvc-8975a.18d451022a5ca6f5 namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 6c8e7c36-03a7-4c79-8ffd-1cf95f844ead reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8975a namespace: kserve-ci-e2e-test resourceVersion: "15664" uid: e562be37-23f7-45ee-a688-6dbea682ae0d kind: Event lastTimestamp: "2026-09-11T16:28:19Z" message: InferenceService [success-200-isvc-8975a] is Ready metadata: creationTimestamp: "2026-09-11T16:28:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:19Z" name: success-200-isvc-8975a.18d45114364aedb3 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 7e22a52b-93eb-4662-8810-5459c5aa1fad reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:28:45.939549Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a6e9aa824 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: 6c67f119-c2c1-41f7-a91a-0719b9afddca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a91afc207 namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: a8954c56-05d1-4343-9721-cf9b40ffc04d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a930a4be3 namespace: kserve-ci-e2e-test resourceVersion: "16041" uid: 92592929-731f-4d17-bfbc-8cfea0784fc0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a96068d2c namespace: kserve-ci-e2e-test resourceVersion: "16042" uid: 6bf61c20-9aec-448e-8aaf-35bb2d0630d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a96809c1c namespace: kserve-ci-e2e-test resourceVersion: "16043" uid: 8cd8a0b3-f0a6-4339-a458-308f4b7bcd65 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a968d2b98 namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: bc1d2149-ce60-495d-8246-89437a83f767 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a9a55da3a namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 65ecdbf6-eb77-493c-825c-885dad36ac18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:28:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:46Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511a9acebf5f namespace: kserve-ci-e2e-test resourceVersion: "16047" uid: 084b0194-a18c-41a8-86cf-a112c15cd892 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:29:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:24Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4511acbbc229a namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: ee1ebcf4-3ed1-4211-a9b8-8c8efb2e60b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4519b9ceeaf6b namespace: kserve-ci-e2e-test resourceVersion: "20349" uid: cc2545aa-e245-44c4-8ff0-5117b5471e02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: b5826f5e-cffe-45f6-a580-69e2aa5c0c7c kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg.18d4519b9cef5ffc namespace: kserve-ci-e2e-test resourceVersion: "20351" uid: 7a843543-9b58-4b2a-92dc-000826723ca3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9c92a-predictor-5b64d8b4b7 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 86912492-2ad0-44c3-995c-fe4c0b48a7b4 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: 'Created pod: success-200-isvc-9c92a-predictor-5b64d8b4b7-rk5wg' metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-9c92a-predictor-5b64d8b4b7.18d4511a6e27d453 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: bb627afe-29e6-473e-96b2-2c6fbc352283 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 9d846e04-df36-41e3-a7f9-5c70efad109b kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Scaled up replica set success-200-isvc-9c92a-predictor-5b64d8b4b7 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-9c92a-predictor.18d4511a69e6d9f9 namespace: kserve-ci-e2e-test resourceVersion: "15960" uid: 874053f7-8231-4dec-a5b0-849a26271324 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 089cd579-25b6-43e5-9e0a-863d68641811 kind: Event lastTimestamp: "2026-09-11T16:29:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:15Z" name: success-200-isvc-9c92a-predictor.18d4511de8cca255 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 89ff5343-3ffc-49e5-94e9-b1ee6e7d84d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15955" uid: 089cd579-25b6-43e5-9e0a-863d68641811 kind: Event lastTimestamp: "2026-09-11T16:29:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:15Z" name: success-200-isvc-9c92a-predictor.18d4511de91e5376 namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 9a82aa49-4f28-4c97-a19e-0bd02c61f1e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:29:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 089cd579-25b6-43e5-9e0a-863d68641811 kind: Event lastTimestamp: "2026-09-11T16:30:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:00Z" name: success-200-isvc-9c92a-predictor.18d45124e6c76fce namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: 3ddbf15b-2f46-4dea-bad0-06f61e2a86fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:29:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9c92a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16206" uid: 089cd579-25b6-43e5-9e0a-863d68641811 kind: Event lastTimestamp: "2026-09-11T16:30:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:00Z" name: success-200-isvc-9c92a-predictor.18d45124e725db12 namespace: kserve-ci-e2e-test resourceVersion: "16961" uid: 0a92a1a1-6827-4b65-b4e4-c7dd92330cde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 5d90636f-dfef-47e4-902e-423fedf19676 kind: Event lastTimestamp: "2026-09-11T16:29:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:34Z" name: success-200-isvc-9c92a.18d4511a69f19e59 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: b6431fe9-2481-475e-ace8-321e3a7003c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 5d90636f-dfef-47e4-902e-423fedf19676 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-9c92a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c92a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-9c92a.18d4511a6a6e47e0 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: b239a547-0ae6-457d-9b66-7493216fb4c0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: 5d90636f-dfef-47e4-902e-423fedf19676 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9c92a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:45Z" name: success-200-isvc-9c92a.18d4511a6a6e91c0 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: 3c1e2f28-0bf4-4a2c-b992-2e61bbb10a36 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 5d90636f-dfef-47e4-902e-423fedf19676 kind: Event lastTimestamp: "2026-09-11T16:29:34Z" message: InferenceService [success-200-isvc-9c92a] is Ready metadata: creationTimestamp: "2026-09-11T16:29:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:34Z" name: success-200-isvc-9c92a.18d45125c296db10 namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: e2123903-facd-4ca9-b623-154112c33175 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:38:00.866218Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a9581-predictor-5656556755-688q4 to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519ba2cf940f namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: a4824c99-be81-408f-bf2b-952761887bae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-a9581-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:01Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519bb1ec5869 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 0f4c4ee5-607e-4e0d-a476-dff07c6039dd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519be8860ab3 namespace: kserve-ci-e2e-test resourceVersion: "20437" uid: db8fb08d-62e6-442c-ba9f-5418a3e699b2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519be9cc6ad0 namespace: kserve-ci-e2e-test resourceVersion: "20439" uid: 9bff5f52-56c0-48bd-8135-b90c959e7215 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519becb43a20 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 670845ae-a05b-47d3-9a3a-1fccd9096964 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519bed282990 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: f41fc801-f109-44a2-a2da-11b7b2436cdf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519bed32b66a namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 363d5b83-f70a-4d5d-821a-d40a62aaa4d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519befd4857f namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 8c55f9f4-48fb-4c88-ba8a-b2ca16b572b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:02Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519bf04e127d namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: 8e19687b-969f-45ab-8839-3326358e4e4e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:38:39Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:39Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d4519c4403d90a namespace: kserve-ci-e2e-test resourceVersion: "20883" uid: 4c42771e-9a4c-469f-8395-2f1fd906f02d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d451abf9df493c namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: 8481329f-43c0-428b-a507-9c3419c04479 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9581-predictor-5656556755-688q4 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 495ada4f-7c74-4cc7-9c2f-12d7daba09f7 kind: Event lastTimestamp: "2026-09-11T16:39:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:11Z" name: success-200-isvc-a9581-predictor-5656556755-688q4.18d451abf9e25cb7 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: d69da643-eb1a-4fbd-bba2-bc4fb9849189 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a9581-predictor-5656556755 namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 134188cd-3c6b-4777-8391-9a98b2f966ff kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'Created pod: success-200-isvc-a9581-predictor-5656556755-688q4' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-a9581-predictor-5656556755.18d4519ba22feeb5 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: d780320e-8e4a-42da-bcec-e0b60937b4e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: dc88e6d2-4529-4a6b-b14c-0b2b91ae95a4 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Scaled up replica set success-200-isvc-a9581-predictor-5656556755 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-a9581-predictor.18d4519b9e1e1257 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: b06d5c8b-62f2-4696-8c32-e610de5886df reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: f73aa7fe-9970-45c3-8627-95904865f16a kind: Event lastTimestamp: "2026-09-11T16:38:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:30Z" name: success-200-isvc-a9581-predictor.18d4519f1ca734aa namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: b7d91cff-f4a7-4643-8e14-e99ec1d1b508 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: f73aa7fe-9970-45c3-8627-95904865f16a kind: Event lastTimestamp: "2026-09-11T16:38:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:30Z" name: success-200-isvc-a9581-predictor.18d4519f1d088a9f namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 485f6bc6-5211-417d-bdc1-61a987867b56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f73aa7fe-9970-45c3-8627-95904865f16a kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: success-200-isvc-a9581-predictor.18d451a61af91436 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: 5604f237-6321-4856-b9ea-93c70b97eb48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:38:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9581-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f73aa7fe-9970-45c3-8627-95904865f16a kind: Event lastTimestamp: "2026-09-11T16:39:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:00Z" name: success-200-isvc-a9581-predictor.18d451a61b688070 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 86fd8d62-6835-4f30-825b-07d237ad6b4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 81d33947-bf10-4b9f-88bf-4841427c5aee kind: Event lastTimestamp: "2026-09-11T16:38:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:49Z" name: success-200-isvc-a9581.18d4519b9e584333 namespace: kserve-ci-e2e-test resourceVersion: "20991" uid: bb23d066-a3e7-4c6f-aec1-d9bd586d2828 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 81d33947-bf10-4b9f-88bf-4841427c5aee kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'Failed to update status for InferenceService "success-200-isvc-a9581": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-a9581.18d4519b9ed2f76f namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 1259e338-cd16-4135-9e29-ae042cde5499 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 81d33947-bf10-4b9f-88bf-4841427c5aee kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9581": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:00Z" name: success-200-isvc-a9581.18d4519b9ed33eec namespace: kserve-ci-e2e-test resourceVersion: "20360" uid: 2127a3cd-d603-4214-906f-dda721da7745 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9581 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 81d33947-bf10-4b9f-88bf-4841427c5aee kind: Event lastTimestamp: "2026-09-11T16:38:49Z" message: InferenceService [success-200-isvc-a9581] is Ready metadata: creationTimestamp: "2026-09-11T16:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:38:49Z" name: success-200-isvc-a9581.18d451a6ffa808cf namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: 36ebf002-5d83-4982-b6dd-4361364a2b2c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:48:26.073955Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ddc34-predictor-54b9f46b47-258vb to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d34189142 namespace: kserve-ci-e2e-test resourceVersion: "25919" uid: 2d2eb7a3-9580-4b5f-9941-b2e86dbd70e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ddc34-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d431b3141 namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 2201a4eb-b1a7-41d3-adb6-09effc0ac827 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d79e4729a namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: c7360a77-318d-46e9-8486-47205911608f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d7b4b296e namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: 65cdce39-4470-475c-9c69-027d25e5cad4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d7efaf397 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: da39d030-f42d-44a0-9a3c-d12489901cc2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d7f7153b2 namespace: kserve-ci-e2e-test resourceVersion: "25975" uid: 078071c2-56ae-452d-96b3-6800c28c811b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d7f7c2027 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: dc256af8-ff46-4ebc-aa5e-b73b1efcfcbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d826d39b5 namespace: kserve-ci-e2e-test resourceVersion: "25978" uid: c0677289-52dd-4071-a809-4e00b794aa94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:48:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:48:27Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d82e4e384 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 58559f1e-ac82-43af-9d0c-5d22f4bce047 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:48:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:49:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:48:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:03Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4522d99c7dc60 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 01572df6-2e7a-4469-acfd-baf5d90a900c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4523d8c8bdf91 namespace: kserve-ci-e2e-test resourceVersion: "26797" uid: a46a9f2d-b215-4dbe-96af-0692d2227358 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:49:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:36Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4523d8c8c7d6a namespace: kserve-ci-e2e-test resourceVersion: "26804" uid: f44b8b46-6592-4562-9b72-f74fc6fee357 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb namespace: kserve-ci-e2e-test resourceVersion: "25914" uid: eebbd631-41d6-4bce-88a0-5958993b067f kind: Event lastTimestamp: "2026-09-11T16:49:38Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:38Z" name: success-200-isvc-ddc34-predictor-54b9f46b47-258vb.18d4523e21e613d1 namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 64ecbd57-a600-4eb6-8541-15b9f6184417 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ddc34-predictor-54b9f46b47 namespace: kserve-ci-e2e-test resourceVersion: "25889" uid: 19b0c21e-0e5f-4170-a40f-a99ff5118d8c kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'Created pod: success-200-isvc-ddc34-predictor-54b9f46b47-258vb' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34-predictor-54b9f46b47.18d4522d3393df33 namespace: kserve-ci-e2e-test resourceVersion: "25912" uid: 1008e881-bec8-4945-baee-5da542cd7f1b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: d60486b7-a47d-4c68-84f0-b3de35fa4dd5 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: Scaled up replica set success-200-isvc-ddc34-predictor-54b9f46b47 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34-predictor.18d4522d3027b558 namespace: kserve-ci-e2e-test resourceVersion: "25890" uid: 048a5e28-76c3-4db3-bcbf-7acc0c25ba79 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 3dae71b6-42e4-4079-8f25-e85190ada4ba kind: Event lastTimestamp: "2026-09-11T16:49:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:11Z" name: success-200-isvc-ddc34-predictor.18d45230ae9ec0b7 namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: efe60108-d9eb-4a6e-bca7-3988abe545f6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 3dae71b6-42e4-4079-8f25-e85190ada4ba kind: Event lastTimestamp: "2026-09-11T16:49:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:11Z" name: success-200-isvc-ddc34-predictor.18d45230af07b9ec namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 8208e7f3-7867-4268-8b06-536462187b50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 3dae71b6-42e4-4079-8f25-e85190ada4ba kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: success-200-isvc-ddc34-predictor.18d452342e1c073c namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: b594d2db-504f-47a6-87ba-298642df3349 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ddc34-predictor namespace: kserve-ci-e2e-test resourceVersion: "26115" uid: 3dae71b6-42e4-4079-8f25-e85190ada4ba kind: Event lastTimestamp: "2026-09-11T16:49:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:49:26Z" name: success-200-isvc-ddc34-predictor.18d452342e7fd1c4 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 9bff041a-8f2f-4bf1-95ef-e4816188ad21 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: e2a8c859-b728-4969-a1be-37c695e95c76 kind: Event lastTimestamp: "2026-09-11T16:49:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:13Z" name: success-200-isvc-ddc34.18d4522d303388cb namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: b675b0a2-71ca-4012-81fd-be7e105db7d4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: e2a8c859-b728-4969-a1be-37c695e95c76 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-ddc34": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ddc34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34.18d4522d30a6d847 namespace: kserve-ci-e2e-test resourceVersion: "25894" uid: 096b4115-d8e1-4c87-a6f4-93a0e22272e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:48:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "25864" uid: e2a8c859-b728-4969-a1be-37c695e95c76 kind: Event lastTimestamp: "2026-09-11T16:48:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ddc34": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:48:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:48:26Z" name: success-200-isvc-ddc34.18d4522d30a71b18 namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: 733dbb1c-2183-417f-a2a2-51da33abdc59 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ddc34 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: e2a8c859-b728-4969-a1be-37c695e95c76 kind: Event lastTimestamp: "2026-09-11T16:49:13Z" message: InferenceService [success-200-isvc-ddc34] is Ready metadata: creationTimestamp: "2026-09-11T16:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:49:13Z" name: success-200-isvc-ddc34.18d45238551c3c42 namespace: kserve-ci-e2e-test resourceVersion: "26532" uid: 4c73ec28-f441-4e8a-ac87-63e7f8ec446d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:38:36.713492Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e5408-predictor-765956d4cf-svlxt to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a3fb79a965 namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: 077d1cf9-d35f-423e-8883-6a1328ac44d0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e5408-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a40b2e5ab1 namespace: kserve-ci-e2e-test resourceVersion: "20827" uid: 5d030844-e6c4-4a4d-9e5b-92c117376c12 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:38:37Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a44150032c namespace: kserve-ci-e2e-test resourceVersion: "20845" uid: b4e6ee8a-4357-49f9-8f08-1e3b5cba9c74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:37Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a442a2945e namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 88dc6e0f-2328-41c8-8fdb-dd4b983bfdcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:37Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a445646e4c namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: 00709ec4-3ac8-4e00-863a-d697950c56bd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:37Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a445d899b7 namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: cb0b0a2e-4dc3-4ac5-8217-19e2ae971c61 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:38:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:37Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a445e2938d namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: fb57f237-62f7-4620-aa60-9d8f8782a97f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a4493a7210 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: 3bd5f43a-0a91-4b82-8303-58d120983405 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a449acd8e4 namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 6676886e-f43e-485e-a3bf-68e2d40209b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:39:14Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:14Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451a471e31b3f namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: fffb3224-b097-4dda-b30f-68e09b42d870 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:46Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451b4507dbf17 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 69cd0a2b-4577-4ced-9a87-8ddf4b0989f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:46Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451b4507e5b8f namespace: kserve-ci-e2e-test resourceVersion: "21737" uid: 98240250-de4f-46e5-8a3d-5c13c6a37070 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5408-predictor-765956d4cf-svlxt namespace: kserve-ci-e2e-test resourceVersion: "20776" uid: d612dee1-fcd2-405b-bbae-a5ef4927efc2 kind: Event lastTimestamp: "2026-09-11T16:39:49Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:49Z" name: success-200-isvc-e5408-predictor-765956d4cf-svlxt.18d451b4fa1dea2f namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: 3848f082-8497-4125-adfa-56cd5d1f3867 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e5408-predictor-765956d4cf namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: c25faf46-58e5-4c5a-aae8-8f27dc371547 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: 'Created pod: success-200-isvc-e5408-predictor-765956d4cf-svlxt' metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-e5408-predictor-765956d4cf.18d451a3fb0589a9 namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: cc281784-54b1-4f06-9b5c-a0a4a40f6b7f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20766" uid: 16b40d19-2e3a-41fd-abe0-cf249fd5419f kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Scaled up replica set success-200-isvc-e5408-predictor-765956d4cf from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-e5408-predictor.18d451a3f9382992 namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: d90f757b-47e2-46d5-b4d6-55c9fe650208 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 17bf3fe7-46c2-4d7a-8786-32155ad2d849 kind: Event lastTimestamp: "2026-09-11T16:38:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:51Z" name: success-200-isvc-e5408-predictor.18d451a777a3ce18 namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 056ea8ee-7f78-440b-9739-6cdfe676b199 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "20767" uid: 17bf3fe7-46c2-4d7a-8786-32155ad2d849 kind: Event lastTimestamp: "2026-09-11T16:38:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:38:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:38:51Z" name: success-200-isvc-e5408-predictor.18d451a778042e59 namespace: kserve-ci-e2e-test resourceVersion: "21010" uid: 0c6ee794-96af-406b-8ff3-e621c856ac2e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:39:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 17bf3fe7-46c2-4d7a-8786-32155ad2d849 kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: success-200-isvc-e5408-predictor.18d451aaf6c03692 namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: a970ebc7-96c1-4246-9919-bf7ae5e229f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-11T16:39:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5408-predictor namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 17bf3fe7-46c2-4d7a-8786-32155ad2d849 kind: Event lastTimestamp: "2026-09-11T16:39:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:36Z" name: success-200-isvc-e5408-predictor.18d451aaf70895d1 namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 77679365-8002-483b-9443-4f9e7da83a97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5408 namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: a404e7c1-b097-4cae-ab3b-b23a1e8fceea kind: Event lastTimestamp: "2026-09-11T16:39:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:24Z" name: success-200-isvc-e5408.18d451a3f93eb393 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: ab536af0-03e6-41c4-8f6b-dc26dd32d86a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5408 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: a404e7c1-b097-4cae-ab3b-b23a1e8fceea kind: Event lastTimestamp: "2026-09-11T16:39:24Z" message: InferenceService [success-200-isvc-e5408] is Ready metadata: creationTimestamp: "2026-09-11T16:39:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:39:24Z" name: success-200-isvc-e5408.18d451af2d87d7d0 namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 7f84b649-97c9-4e76-bfd4-9127226f3b17 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:39:46.985255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:39:46Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b457ffabc3 namespace: kserve-ci-e2e-test resourceVersion: "21769" uid: c5ac41d6-7565-4fb3-8f98-499ad0fc7767 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ea3c5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:47Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b463714f6d namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: 414f71fb-3a38-4a19-9150-e1820c006232 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b49d88be68 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: 0d957c48-bb96-4377-8e0f-7ebc123ed798 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b49ee0c57d namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 3a835441-fb83-45e4-8f80-6a083237f2ae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4a26c2ca4 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 24117d4e-60b6-4744-90fe-ed4cd809502c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4a2ddd72b namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: a27d73bb-c194-4da9-97d5-869cbe788e9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4a2e76b36 namespace: kserve-ci-e2e-test resourceVersion: "21830" uid: 6963050d-ec50-48a7-be36-9b21cff1d1c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4a594d861 namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: a70422c1-4d62-484f-b7b5-d9cb8f1769f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:39:48Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4a6067b21 namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: cf1f2abc-e078-4cdf-8cd0-40c392015808 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:49:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:04Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d451b4cdbdbf32 namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: 033f845b-7777-494b-82bc-4950b6da95a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:01Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d4523580493471 namespace: kserve-ci-e2e-test resourceVersion: "26319" uid: 340ba034-944d-4f75-949b-35e311d031f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:49:01Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:01Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d452358049fbc1 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: 77651151-28f7-4802-8e68-7b292896b961 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:49:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: cce281a8-aa85-484f-822a-4115ca0e68fa kind: Event lastTimestamp: "2026-09-11T16:49:04Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:49:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:49:04Z" name: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh.18d4523642222568 namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 53e45a50-da80-473f-a5f0-63f2a12a2d38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ea3c5-predictor-86955dc77c namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 63f859fe-8386-4f49-80dd-100c6e905715 kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: 'Created pod: success-200-isvc-ea3c5-predictor-86955dc77c-sd4rh' metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:46Z" name: success-200-isvc-ea3c5-predictor-86955dc77c.18d451b457827f2c namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 7783ca64-6d5e-4623-aebb-9f6dcb935a7a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21757" uid: 490bf382-1e0f-4513-8135-a7aef9da537c kind: Event lastTimestamp: "2026-09-11T16:39:46Z" message: Scaled up replica set success-200-isvc-ea3c5-predictor-86955dc77c from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:39:46Z" name: success-200-isvc-ea3c5-predictor.18d451b455b1bf83 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: dde379c0-a89c-424f-9ff4-373c7ccadf00 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: e2be38c2-2739-4111-9729-3ebd1d42b8e1 kind: Event lastTimestamp: "2026-09-11T16:40:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:16Z" name: success-200-isvc-ea3c5-predictor.18d451b7d4258139 namespace: kserve-ci-e2e-test resourceVersion: "22137" uid: 46d2a828-03d6-49fb-9b2d-ff5473be7892 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: e2be38c2-2739-4111-9729-3ebd1d42b8e1 kind: Event lastTimestamp: "2026-09-11T16:40:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:40:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:16Z" name: success-200-isvc-ea3c5-predictor.18d451b7d4904f9f namespace: kserve-ci-e2e-test resourceVersion: "22138" uid: 4c9de2f5-8f25-42ba-85df-607c7fd4a68d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: e2be38c2-2739-4111-9729-3ebd1d42b8e1 kind: Event lastTimestamp: "2026-09-11T16:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:47Z" name: success-200-isvc-ea3c5-predictor.18d451bed26f952b namespace: kserve-ci-e2e-test resourceVersion: "22408" uid: b209f673-4ec1-4c9f-bc76-17711a3e99a0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-11T16:40:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ea3c5-predictor namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: e2be38c2-2739-4111-9729-3ebd1d42b8e1 kind: Event lastTimestamp: "2026-09-11T16:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:40:47Z" name: success-200-isvc-ea3c5-predictor.18d451bed2cfe124 namespace: kserve-ci-e2e-test resourceVersion: "22410" uid: 4e05228d-9669-4aed-acac-5b44f3af1498 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:39:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: b0bde4b2-88ee-4592-94db-0ad114466c3f kind: Event lastTimestamp: "2026-09-11T16:40:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:35Z" name: success-200-isvc-ea3c5.18d451b455b6bde5 namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: 3ad40e6d-712b-4e91-b999-797e12ebd72b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ea3c5 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: b0bde4b2-88ee-4592-94db-0ad114466c3f kind: Event lastTimestamp: "2026-09-11T16:40:35Z" message: InferenceService [success-200-isvc-ea3c5] is Ready metadata: creationTimestamp: "2026-09-11T16:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:40:35Z" name: success-200-isvc-ea3c5.18d451bf88da02ac namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 4c691b4f-661a-451f-9492-45588935ad6a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:29:21.773725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn to ip-10-0-136-18.ec2.internal metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:29:21Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122c67cce12 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 2a51607b-c645-4595-be64-0ce362f21aa4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122e85b55f7 namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: 72680873-4c6a-4cf2-abb8-0d82650fe0c3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1928" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122e999657a namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 23745b12-8bb3-4961-bc8e-81018fffc02a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122ec4a3116 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: d3933882-027a-4864-b269-53e68e8c1b2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122ecafd6fe namespace: kserve-ci-e2e-test resourceVersion: "16558" uid: 4ca105f1-45a9-40bb-9f58-7303aceaf1a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122ecb8f709 namespace: kserve-ci-e2e-test resourceVersion: "16559" uid: 639510f9-2c79-4bdb-9134-06aa6fcfe613 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122ef6acbdb namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: d2b99fd4-6f5d-42ca-915b-9e357c3b8347 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:29:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:22Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122f0119dff namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 8c97ada4-2a39-4b03-b79b-8b41e7817be3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-11T16:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d45122fb1895da namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: b81ff01a-7e6a-4576-a217-02478fdbd176 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d451a3f5d86dea namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 5091e752-b67a-4f68-b6e3-15f0a2a6d43c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:38:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-11T16:38:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:36Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d451a3f5d90caa namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 61045643-c80f-4aa3-801d-cb5813984553 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 08575857-6e56-4643-98c3-e7947e5c1264 kind: Event lastTimestamp: "2026-09-11T16:38:38Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-11T16:38:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:38Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn.18d451a46fbfe0fd namespace: kserve-ci-e2e-test resourceVersion: "20871" uid: 813dfc81-df33-41c0-a258-7296b8379913 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-18.ec2.internal source: component: kubelet host: ip-10-0-136-18.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ff919-predictor-6bbdbd4fc7 namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: eb9a4127-3946-4fc2-aaeb-1208175e89ae kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'Created pod: success-200-isvc-ff919-predictor-6bbdbd4fc7-7vgxn' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:21Z" name: success-200-isvc-ff919-predictor-6bbdbd4fc7.18d45122c5f2a76f namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: 6437b02c-f8f4-4232-b855-0c8ee8d2f6c1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: dff62e38-8b26-4546-a32e-be45c385556f kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: Scaled up replica set success-200-isvc-ff919-predictor-6bbdbd4fc7 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:21Z" name: success-200-isvc-ff919-predictor.18d45122c4247005 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: bcf509c0-b442-4c3a-85ac-ba2be84c1c3d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: c54b646f-6884-4ff9-baa8-f5faa7de3dcf kind: Event lastTimestamp: "2026-09-11T16:29:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:36Z" name: success-200-isvc-ff919-predictor.18d4512642a838b1 namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: f268855c-195c-4c27-9e24-299b56dc3991 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: c54b646f-6884-4ff9-baa8-f5faa7de3dcf kind: Event lastTimestamp: "2026-09-11T16:29:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:29:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:36Z" name: success-200-isvc-ff919-predictor.18d45126430613d9 namespace: kserve-ci-e2e-test resourceVersion: "16725" uid: 7ea2fa07-8e79-4f6e-9a48-8318d1a60440 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:29:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: c54b646f-6884-4ff9-baa8-f5faa7de3dcf kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: success-200-isvc-ff919-predictor.18d45129c1ab8cee namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 61793eb9-a89c-4521-8ba9-5c21df2c1e88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-11T16:29:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ff919-predictor namespace: kserve-ci-e2e-test resourceVersion: "16724" uid: c54b646f-6884-4ff9-baa8-f5faa7de3dcf kind: Event lastTimestamp: "2026-09-11T16:30:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:29:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:36Z" name: success-200-isvc-ff919-predictor.18d45129c1f92598 namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 47e33bdd-761c-4f4b-8c92-3e4638216f95 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16438" uid: 54af4f57-d160-45f4-8d17-908a36193258 kind: Event lastTimestamp: "2026-09-11T16:30:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:08Z" name: success-200-isvc-ff919.18d45122c43e3e44 namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 0096bb2d-73d7-4351-8591-66b93c2fbf64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 54af4f57-d160-45f4-8d17-908a36193258 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-ff919": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:21Z" name: success-200-isvc-ff919.18d45122ce0fb628 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 53e0ba77-7892-4e4c-87bf-cf0a5fa6ac85 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "16476" uid: 54af4f57-d160-45f4-8d17-908a36193258 kind: Event lastTimestamp: "2026-09-11T16:29:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ff919": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-11T16:29:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:21Z" name: success-200-isvc-ff919.18d45122ce0ffec6 namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: 65eef0af-daf8-4106-9f2e-e00a7aae52b1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ff919 namespace: kserve-ci-e2e-test resourceVersion: "17030" uid: 54af4f57-d160-45f4-8d17-908a36193258 kind: Event lastTimestamp: "2026-09-11T16:30:08Z" message: InferenceService [success-200-isvc-ff919] is Ready metadata: creationTimestamp: "2026-09-11T16:30:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:30:08Z" name: success-200-isvc-ff919.18d4512db6b23165 namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: 5ab8b9b9-b9ba-418b-a3d0-d7df3c8181c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:50:01.854527Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-482a1-556896489f-gxlb5 to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:50:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:50:01Z" name: switch-graph-482a1-556896489f-gxlb5.18d4524381102e48 namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: efeb4c2b-2d11-4d9a-960d-7e83c382c573 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:02Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T16:50:02Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:50:02Z" name: switch-graph-482a1-556896489f-gxlb5.18d45243a302d117 namespace: kserve-ci-e2e-test resourceVersion: "27155" uid: 23d65ed3-db24-4c55-8ed1-fb949c432ad9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-482a1} kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T16:50:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:02Z" name: switch-graph-482a1-556896489f-gxlb5.18d45243a581698c namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: 17536ad1-b206-48bd-88cc-f59b966aa42b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-482a1} kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T16:50:02Z" message: 'Created container: switch-graph-482a1' metadata: creationTimestamp: "2026-09-11T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:02Z" name: switch-graph-482a1-556896489f-gxlb5.18d45243a849482a namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: 38e66c6d-8b64-4bee-82e9-646591d45bec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-482a1} kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T16:50:02Z" message: Started container switch-graph-482a1 metadata: creationTimestamp: "2026-09-11T16:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:50:02Z" name: switch-graph-482a1-556896489f-gxlb5.18d45243a8cd48f4 namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: 92f89bb5-802c-4de1-9e22-869d391e6939 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T17:06:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-482a1} kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T17:06:21Z" message: Stopping container switch-graph-482a1 metadata: creationTimestamp: "2026-09-11T17:06:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:21Z" name: switch-graph-482a1-556896489f-gxlb5.18d453277c4b3fe9 namespace: kserve-ci-e2e-test resourceVersion: "34044" uid: 320d3e16-2392-4597-9eb1-571b52ce4e9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-11T17:06:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-482a1} kind: Pod name: switch-graph-482a1-556896489f-gxlb5 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 1b8a1193-a8e5-4673-8e7a-7a7ca558b923 kind: Event lastTimestamp: "2026-09-11T17:06:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T17:06:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T17:06:44Z" name: switch-graph-482a1-556896489f-gxlb5.18d453284d5756f6 namespace: kserve-ci-e2e-test resourceVersion: "34293" uid: 7356c7ee-62dc-4695-a5f2-07dbd91e00f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-482a1-556896489f namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: 95b23615-8d8b-44ad-815b-37628ded516f kind: Event lastTimestamp: "2026-09-11T16:50:01Z" message: 'Created pod: switch-graph-482a1-556896489f-gxlb5' metadata: creationTimestamp: "2026-09-11T16:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:01Z" name: switch-graph-482a1-556896489f.18d4524380acc225 namespace: kserve-ci-e2e-test resourceVersion: "27132" uid: 1ea7d060-f0dc-4729-827c-177fabf9b787 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 3412023b-1219-4e77-9a7a-237bb1cc147b kind: Event lastTimestamp: "2026-09-11T16:50:01Z" message: Scaled up replica set switch-graph-482a1-556896489f from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:01Z" name: switch-graph-482a1.18d452437ff567d4 namespace: kserve-ci-e2e-test resourceVersion: "27128" uid: e9c98f10-17d4-428d-90d3-258f558cf6cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27144" uid: 29d23e2e-d176-4814-b3dd-17651c686add kind: Event lastTimestamp: "2026-09-11T16:50:01Z" message: InferenceGraph [switch-graph-482a1] is Ready metadata: creationTimestamp: "2026-09-11T16:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:50:01Z" name: switch-graph-482a1.18d452438579dc21 namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: b89c1b06-1f68-46b1-aa25-6604cefdf77f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: fe778241-2c99-4eb9-812d-4c553f0a462d kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: switch-graph-482a1.18d45246fed97093 namespace: kserve-ci-e2e-test resourceVersion: "27279" uid: 45939f84-c123-479c-b406-f0300cdaaefb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: fe778241-2c99-4eb9-812d-4c553f0a462d kind: Event lastTimestamp: "2026-09-11T16:50:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:16Z" name: switch-graph-482a1.18d45246ff3f3626 namespace: kserve-ci-e2e-test resourceVersion: "27281" uid: 99846a7a-6f59-4870-9206-5791ce3fd75e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: fe778241-2c99-4eb9-812d-4c553f0a462d kind: Event lastTimestamp: "2026-09-11T16:50:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:31Z" name: switch-graph-482a1.18d4524a7e3f34a6 namespace: kserve-ci-e2e-test resourceVersion: "27410" uid: 796ac330-1ca9-4493-867f-97a4e803468c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:50:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-482a1 namespace: kserve-ci-e2e-test resourceVersion: "27280" uid: fe778241-2c99-4eb9-812d-4c553f0a462d kind: Event lastTimestamp: "2026-09-11T16:50:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:50:31Z" name: switch-graph-482a1.18d4524a7ea4e41d namespace: kserve-ci-e2e-test resourceVersion: "27411" uid: 95302d89-84fe-4667-931b-ebbb5eddaae6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:28:31.489028Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-8975a-67db9b666-tz2kd to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:28:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:28:31Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511711492570 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: b25de661-cab9-4637-bec4-f3a5e6b62bc2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-8975a-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:31Z" name: switch-graph-8975a-67db9b666-tz2kd.18d451171bf880be namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 9faa65eb-31af-473e-a79f-475540d5b955 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15778" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:32Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:28:32Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511756d3f8a2 namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: 8ebfbe48-e3e7-4f56-bd16-c6ef8796c372 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:32Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" metadata: creationTimestamp: "2026-09-11T16:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:32Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511757f47210 namespace: kserve-ci-e2e-test resourceVersion: "15808" uid: 39afcdc3-1b3b-46ca-a6cc-4487a6613062 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" in 3.172s (3.172s including waiting). Image size: 214907987 bytes.' metadata: creationTimestamp: "2026-09-11T16:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:35Z" name: switch-graph-8975a-67db9b666-tz2kd.18d45118150f8708 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: c4f2e024-f709-4755-9109-1d9e63960929 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:35Z" message: 'Created container: switch-graph-8975a' metadata: creationTimestamp: "2026-09-11T16:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:35Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511818076f0d namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: ad378eb6-6fa5-4b3a-acdd-e1646d4fdef1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:35Z" message: Started container switch-graph-8975a metadata: creationTimestamp: "2026-09-11T16:28:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:35Z" name: switch-graph-8975a-67db9b666-tz2kd.18d45118187edfca namespace: kserve-ci-e2e-test resourceVersion: "15831" uid: cac55abc-a711-4a1c-8122-d457cdf87739 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:28:45Z" message: Stopping container switch-graph-8975a metadata: creationTimestamp: "2026-09-11T16:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:28:45Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511a5fdf1e3c namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: c1d7cf95-d722-4697-ad65-8cc7c6e12123 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-8975a} kind: Pod name: switch-graph-8975a-67db9b666-tz2kd namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: b1e63add-64d3-426b-86a5-f0c99bf23920 kind: Event lastTimestamp: "2026-09-11T16:29:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:12Z" name: switch-graph-8975a-67db9b666-tz2kd.18d4511ad84e50a0 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 86ade833-e315-4524-b129-66b38701403a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-8975a-67db9b666 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 5af8abc5-ac3b-4a24-a941-8ff13d354fa6 kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: 'Created pod: switch-graph-8975a-67db9b666-tz2kd' metadata: creationTimestamp: "2026-09-11T16:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: switch-graph-8975a-67db9b666.18d4511710ddccb7 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: c03458f3-6e2c-4ea7-8e49-cad3c1396831 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-8975a namespace: kserve-ci-e2e-test resourceVersion: "15764" uid: 94d01e15-b7ed-45c5-8ba3-d8ea89cc8fbb kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: Scaled up replica set switch-graph-8975a-67db9b666 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:28:31Z" name: switch-graph-8975a.18d451170fe2077f namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 97c46356-77b0-413a-b0db-78cb77df7341 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:28:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-8975a namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 0fc7c76b-cdd5-4bf1-8f39-e1a35accd48d kind: Event lastTimestamp: "2026-09-11T16:28:31Z" message: InferenceGraph [switch-graph-8975a] is Ready metadata: creationTimestamp: "2026-09-11T16:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:28:31Z" name: switch-graph-8975a.18d45117150f8491 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 10a3662d-a1ef-4893-86fe-1e849ec10420 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-11T16:29:45.953827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16810" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9c92a-7458554479-n8fnv to ip-10-0-132-119.ec2.internal metadata: creationTimestamp: "2026-09-11T16:29:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-11T16:29:45Z" name: switch-graph-9c92a-7458554479-n8fnv.18d4512867bbd1f0 namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 8fa62cce-49c2-4209-b19c-972df79c83ba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6c989467fc-schgw source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:29:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9c92a-serving-cert" not found' metadata: creationTimestamp: "2026-09-11T16:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:46Z" name: switch-graph-9c92a-7458554479-n8fnv.18d4512874ada845 namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: 0ea151e1-21c6-461e-a1ca-f11af7d7ee80 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:47Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:29:47Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-11T16:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-11T16:29:47Z" name: switch-graph-9c92a-7458554479-n8fnv.18d45128ad6c77a5 namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 3360e9f2-4222-49d4-a621-47adf11d5435 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c92a} kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:29:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:7ae24018e4b762b3a4204d7b989ec2ec72d61978201f42380712a0f4d411d8ad" already present on machine metadata: creationTimestamp: "2026-09-11T16:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:47Z" name: switch-graph-9c92a-7458554479-n8fnv.18d45128aeb6fc95 namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: 856f4f48-c382-48b3-85d2-358a24f4fdad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c92a} kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:29:47Z" message: 'Created container: switch-graph-9c92a' metadata: creationTimestamp: "2026-09-11T16:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:47Z" name: switch-graph-9c92a-7458554479-n8fnv.18d45128b16d4d2c namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 701c4fcc-973c-47af-a301-d3fae07a0cb4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c92a} kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:29:47Z" message: Started container switch-graph-9c92a metadata: creationTimestamp: "2026-09-11T16:29:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:29:47Z" name: switch-graph-9c92a-7458554479-n8fnv.18d45128b1df2813 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: dbf4b7ed-869a-4da7-86ef-1a488b3bd483 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c92a} kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:38:00Z" message: Stopping container switch-graph-9c92a metadata: creationTimestamp: "2026-09-11T16:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:00Z" name: switch-graph-9c92a-7458554479-n8fnv.18d4519b957c5fe9 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: d60af4dd-18d8-4fda-9630-5b9bab97eb55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-11T16:38:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9c92a} kind: Pod name: switch-graph-9c92a-7458554479-n8fnv namespace: kserve-ci-e2e-test resourceVersion: "16812" uid: 473a3212-1952-40cc-8afa-dafe84b489f1 kind: Event lastTimestamp: "2026-09-11T16:38:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-11T16:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-11T16:38:28Z" name: switch-graph-9c92a-7458554479-n8fnv.18d4519c5d12aad2 namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 702d049d-d92b-4e4e-bfa0-86e143ec7567 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-119.ec2.internal source: component: kubelet host: ip-10-0-132-119.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9c92a-7458554479 namespace: kserve-ci-e2e-test resourceVersion: "16806" uid: ba20e368-4aad-4546-8c75-898aebd0bb13 kind: Event lastTimestamp: "2026-09-11T16:29:45Z" message: 'Created pod: switch-graph-9c92a-7458554479-n8fnv' metadata: creationTimestamp: "2026-09-11T16:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:45Z" name: switch-graph-9c92a-7458554479.18d45128674ffac3 namespace: kserve-ci-e2e-test resourceVersion: "16811" uid: 2c631db1-0a48-426b-85f0-015c2127e39c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 9a656ecc-eeeb-42b6-ae0c-2e826865f661 kind: Event lastTimestamp: "2026-09-11T16:29:45Z" message: Scaled up replica set switch-graph-9c92a-7458554479 from 0 to 1 metadata: creationTimestamp: "2026-09-11T16:29:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:29:45Z" name: switch-graph-9c92a.18d4512866618630 namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 4aa1f6f1-7c17-4f2e-a21f-35d9a3b74ae7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:29:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 002adf6c-6d84-4ef8-87cc-43b7c03c96d7 kind: Event lastTimestamp: "2026-09-11T16:29:46Z" message: InferenceGraph [switch-graph-9c92a] is Ready metadata: creationTimestamp: "2026-09-11T16:29:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-11T16:29:46Z" name: switch-graph-9c92a.18d451286bb22f44 namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 6ba4727f-c8eb-411e-bb4c-b37c85cb8c40 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 40820cb2-571d-41f0-a788-0ad45bcdecb7 kind: Event lastTimestamp: "2026-09-11T16:30:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:00Z" name: switch-graph-9c92a.18d4512be4ef9b60 namespace: kserve-ci-e2e-test resourceVersion: "16958" uid: 0ebd395a-eeab-4fae-a4d4-c5d00c60ecaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: 40820cb2-571d-41f0-a788-0ad45bcdecb7 kind: Event lastTimestamp: "2026-09-11T16:30:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-11T16:30:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:00Z" name: switch-graph-9c92a.18d4512be532bc2a namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: d0bb778b-7e4c-43ae-9fbe-32c149f4f2b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 40820cb2-571d-41f0-a788-0ad45bcdecb7 kind: Event lastTimestamp: "2026-09-11T16:30:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:15Z" name: switch-graph-9c92a.18d4512f63d0fc21 namespace: kserve-ci-e2e-test resourceVersion: "17092" uid: eb815507-5425-4d04-b0f5-ffe9d76d50b3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-11T16:30:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-9c92a namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 40820cb2-571d-41f0-a788-0ad45bcdecb7 kind: Event lastTimestamp: "2026-09-11T16:30:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-11T16:30:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-11T16:30:15Z" name: switch-graph-9c92a.18d4512f64443488 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 59dbdad4-4ca8-468a-babe-deb85b5bb053 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34322"