--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:49:23.881151Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22094" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4c9b0-754cff594-62dgq to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:49:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:49:23Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f535659063a namespace: kserve-ci-e2e-test resourceVersion: "22100" uid: b22bf794-d4c1-4388-a29f-12b44d9ea8f7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:49:24Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:49:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-4c9b0-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:49:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:24Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f535eaf426b namespace: kserve-ci-e2e-test resourceVersion: "22116" uid: ebc26279-37cd-4291-94d6-64e560bccfb7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:25Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22104" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:49:25Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:49:25Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f53c93aabe1 namespace: kserve-ci-e2e-test resourceVersion: "22125" uid: 91f2705a-310d-4112-8b6d-2f61c94e3b65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4c9b0} kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:49:25Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:25Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f53ca71a6e4 namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: f2df18c2-16f7-43ee-9bf7-ec3495a9bca2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4c9b0} kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:49:25Z" message: 'Created container: ensemble-graph-4c9b0' metadata: creationTimestamp: "2026-07-21T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:25Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f53cd6fb51c namespace: kserve-ci-e2e-test resourceVersion: "22128" uid: 4a26418c-498f-4591-8959-6665059fe61d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4c9b0} kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:49:25Z" message: Started container ensemble-graph-4c9b0 metadata: creationTimestamp: "2026-07-21T17:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:25Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45f53cdebca91 namespace: kserve-ci-e2e-test resourceVersion: "22130" uid: 50ab6008-7c6d-4428-b156-355240f76128 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4c9b0} kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Stopping container ensemble-graph-4c9b0 metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45fc689858709 namespace: kserve-ci-e2e-test resourceVersion: "25903" uid: 26ccbe5f-4a08-419a-bcaa-7d6da9a66043 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4c9b0} kind: Pod name: ensemble-graph-4c9b0-754cff594-62dgq namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 12d23a8f-707c-4ff1-8719-b1b33a5c134b kind: Event lastTimestamp: "2026-07-21T17:58:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:07Z" name: ensemble-graph-4c9b0-754cff594-62dgq.18c45fc7653c1091 namespace: kserve-ci-e2e-test resourceVersion: "26410" uid: b36e6ca5-87e7-409d-acfb-c4572ad5d491 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4c9b0-754cff594 namespace: kserve-ci-e2e-test resourceVersion: "22091" uid: 40e1a3ce-3c3b-4fcf-bf0f-500c95cd9c5e kind: Event lastTimestamp: "2026-07-21T17:49:23Z" message: 'Created pod: ensemble-graph-4c9b0-754cff594-62dgq' metadata: creationTimestamp: "2026-07-21T17:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:23Z" name: ensemble-graph-4c9b0-754cff594.18c45f5355ff3c1e namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: 6cff2d75-d42f-4d53-bb67-30eeb67c8b5d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 1417c44f-8c19-4db7-8099-b7a8c84742ff kind: Event lastTimestamp: "2026-07-21T17:49:23Z" message: Scaled up replica set ensemble-graph-4c9b0-754cff594 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:23Z" name: ensemble-graph-4c9b0.18c45f535538d9eb namespace: kserve-ci-e2e-test resourceVersion: "22092" uid: b1990241-d9db-408f-83e6-7395959690ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22106" uid: 25d8a1f0-6f18-4434-befb-3ab658b481ef kind: Event lastTimestamp: "2026-07-21T17:49:23Z" message: InferenceGraph [ensemble-graph-4c9b0] is Ready metadata: creationTimestamp: "2026-07-21T17:49:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:23Z" name: ensemble-graph-4c9b0.18c45f535a89a0ef namespace: kserve-ci-e2e-test resourceVersion: "22108" uid: 190c3551-07f6-40bb-bb95-51c5647d2ffb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 5549e65f-10e8-4e18-932c-5fa08fdc463c kind: Event lastTimestamp: "2026-07-21T17:49:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:38Z" name: ensemble-graph-4c9b0.18c45f56d46fd9e6 namespace: kserve-ci-e2e-test resourceVersion: "22248" uid: d73e7884-26b5-48d7-902f-6e954146b5da reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: 5549e65f-10e8-4e18-932c-5fa08fdc463c kind: Event lastTimestamp: "2026-07-21T17:49:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:38Z" name: ensemble-graph-4c9b0.18c45f56d4df6ef4 namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: ff2dc9f2-cdf0-4332-9e05-a98c4e2f3af2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: 5549e65f-10e8-4e18-932c-5fa08fdc463c kind: Event lastTimestamp: "2026-07-21T17:49:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:53Z" name: ensemble-graph-4c9b0.18c45f5a53ce8d40 namespace: kserve-ci-e2e-test resourceVersion: "22405" uid: 2c5d5a35-6ef4-45a1-bdef-44f14349124b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: 5549e65f-10e8-4e18-932c-5fa08fdc463c kind: Event lastTimestamp: "2026-07-21T17:49:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:53Z" name: ensemble-graph-4c9b0.18c45f5a544ca523 namespace: kserve-ci-e2e-test resourceVersion: "22407" uid: d40a73c1-e01f-4a76-b2bb-d6a67d914a94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:03.575057Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-f64bd-7d86669d75-5pcrl to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:03Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40a3bb467e namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: f00dba64-363b-4f00-a148-24edf4676cfa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:03Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-f64bd-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:03Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40aeea59ad namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: 5992efa6-76b1-40ac-bea2-d105799ef36a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:04Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:04Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40e0f7a275 namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 3b7f27c6-7823-407e-84cf-eb00136e990b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f64bd} kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:04Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40e22b8e05 namespace: kserve-ci-e2e-test resourceVersion: "21076" uid: e04a7bb7-29a4-4fbb-b419-81c85f7db0e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f64bd} kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:04Z" message: 'Created container: ensemble-graph-f64bd' metadata: creationTimestamp: "2026-07-21T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:04Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40e4e91348 namespace: kserve-ci-e2e-test resourceVersion: "21077" uid: 89eff0bf-25aa-4e2d-b773-9d4c28124dbb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f64bd} kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:04Z" message: Started container ensemble-graph-f64bd metadata: creationTimestamp: "2026-07-21T17:48:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:04Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f40e55d182d namespace: kserve-ci-e2e-test resourceVersion: "21078" uid: 98c4332e-489f-4009-beb2-911d176e174d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f64bd} kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Stopping container ensemble-graph-f64bd metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:13Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f42fb2bcfae namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 7370ef26-6365-45bf-9ff6-759f7d00e7b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f64bd} kind: Pod name: ensemble-graph-f64bd-7d86669d75-5pcrl namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 3d51e87a-c1e1-4869-8874-c868f839747d kind: Event lastTimestamp: "2026-07-21T17:48:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:40Z" name: ensemble-graph-f64bd-7d86669d75-5pcrl.18c45f4387056fa6 namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 68f2d05d-2ba9-4507-815f-c1b36c3312d9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-f64bd-7d86669d75 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 01d4953c-f9b8-4d5c-8e8a-7d0b51d7acf1 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'Created pod: ensemble-graph-f64bd-7d86669d75-5pcrl' metadata: creationTimestamp: "2026-07-21T17:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: ensemble-graph-f64bd-7d86669d75.18c45f40a35d2d5e namespace: kserve-ci-e2e-test resourceVersion: "21042" uid: b9453609-1d5d-44b6-af3f-ffdb443131fe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-f64bd namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: 640736f7-9416-40b3-b69c-69e1ef088868 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: Scaled up replica set ensemble-graph-f64bd-7d86669d75 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: ensemble-graph-f64bd.18c45f40a23df98e namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e098373a-4ef3-40d8-8526-59fbd6d581cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f64bd namespace: kserve-ci-e2e-test resourceVersion: "21055" uid: 49bfb7fd-3b9b-4101-b24a-6ccc05d30f24 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: InferenceGraph [ensemble-graph-f64bd] is Ready metadata: creationTimestamp: "2026-07-21T17:48:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:03Z" name: ensemble-graph-f64bd.18c45f40a7e3bf55 namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: f8535357-c275-4073-a73a-cebbece9adec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:13.927468Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21248" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f430cc87fff namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: 816504a0-1e16-4836-819f-f90c922e3a93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432747cd4e namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 4e03edf9-f358-4918-a982-8d4ee926e03a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f4328986616 namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: 2a468e2b-3ef2-48c9-8e18-0072ae62226a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432ada03ef namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: f06605ca-d063-49b5-86b8-b3f8e3b1e8f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432b65cf12 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 3de4a48a-00bd-4360-a29f-a48c30dbbdec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432b6f4351 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 2b55ea3e-5f5c-4375-8d27-e7c4ab828693 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432eb4d78c namespace: kserve-ci-e2e-test resourceVersion: "21285" uid: 9a9b0be2-8f94-4fcf-b949-702504fb0c15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f432f295f24 namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 9213600f-a87b-4235-95fb-dd3c69c91d7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:57:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:41Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45f434e70d3eb namespace: kserve-ci-e2e-test resourceVersion: "26042" uid: 072d9de3-a5fa-45b7-bac8-3247973a9cc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45fc696ac9f81 namespace: kserve-ci-e2e-test resourceVersion: "25976" uid: 230ff4e0-c389-420c-8d69-aebcc2779960 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45fc696ad16ca namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: fa1431a3-6e7e-49ef-9370-c6cffeb827ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr namespace: kserve-ci-e2e-test resourceVersion: "21250" uid: 1633ad8f-0036-4ccf-893f-61ffdbb15b56 kind: Event lastTimestamp: "2026-07-21T17:57:41Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:41Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr.18c45fc716e1f8d6 namespace: kserve-ci-e2e-test resourceVersion: "26040" uid: 907029a8-c6e8-4ba3-9308-2e4fcd3f7931 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4c9b0-predictor-7fc4f48954 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 4e5d208e-96f6-475b-93ea-69b00625fc70 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'Created pod: error-404-isvc-4c9b0-predictor-7fc4f48954-k7lmr' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-4c9b0-predictor-7fc4f48954.18c45f430c54a500 namespace: kserve-ci-e2e-test resourceVersion: "21249" uid: 3dec49d4-f608-4e03-b6df-c5dec6300f11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: b7edf952-16b0-4d3c-9e75-f349d4240041 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Scaled up replica set error-404-isvc-4c9b0-predictor-7fc4f48954 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-4c9b0-predictor.18c45f430a9286a5 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: e841bec0-236f-405b-97fb-d0c5be28d929 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: b5624e34-741a-4f78-ab1c-d1b0279e0a20 kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: error-404-isvc-4c9b0-predictor.18c45f46891d864e namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 40836a4d-a6ca-4f31-867c-ec7660aa672f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: b5624e34-741a-4f78-ab1c-d1b0279e0a20 kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: error-404-isvc-4c9b0-predictor.18c45f46898725be namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: df9461ae-be77-4d0d-905d-a7975eb2ee6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: b5624e34-741a-4f78-ab1c-d1b0279e0a20 kind: Event lastTimestamp: "2026-07-21T17:49:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:43Z" name: error-404-isvc-4c9b0-predictor.18c45f4a0827e2dc namespace: kserve-ci-e2e-test resourceVersion: "22316" uid: 2c47dcfb-a03e-44b6-8ec7-4bcd50756dd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: b5624e34-741a-4f78-ab1c-d1b0279e0a20 kind: Event lastTimestamp: "2026-07-21T17:49:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:43Z" name: error-404-isvc-4c9b0-predictor.18c45f4a088e1996 namespace: kserve-ci-e2e-test resourceVersion: "22319" uid: 70ceef94-3541-4cde-9156-96cc078cc64f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: b51b387c-fc99-42e1-99d0-eb14bbdf7fe5 kind: Event lastTimestamp: "2026-07-21T17:49:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:11Z" name: error-404-isvc-4c9b0.18c45f430ac361dc namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: dbcc8692-5216-4ded-8ce3-9f9b08db68e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: b51b387c-fc99-42e1-99d0-eb14bbdf7fe5 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'Failed to update status for InferenceService "error-404-isvc-4c9b0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4c9b0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-4c9b0.18c45f430b46b345 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: f0c21857-52ee-4b1f-87fc-36aab2b66f4d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: b51b387c-fc99-42e1-99d0-eb14bbdf7fe5 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4c9b0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-4c9b0.18c45f430b47113f namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 550000d2-fcb3-4587-ab0c-d5c687a51bb1 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: b51b387c-fc99-42e1-99d0-eb14bbdf7fe5 kind: Event lastTimestamp: "2026-07-21T17:49:11Z" message: InferenceService [error-404-isvc-4c9b0] is Ready metadata: creationTimestamp: "2026-07-21T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:11Z" name: error-404-isvc-4c9b0.18c45f505da4c240 namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 70acd333-d3d2-40aa-8d0e-7e336e37f6cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:58.670386Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14760" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:58Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e97dc0bcc06 namespace: kserve-ci-e2e-test resourceVersion: "14765" uid: 4284884f-6653-4451-ab87-68a3cb8ec88b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4dfe2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:58Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e97e5147e2f namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: 98bc1037-157e-49ec-9b61-258bbf8dddbc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:59Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e982060c3ce namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 59df1a10-291a-4fd0-8b6b-a2b04b32f8d1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e982265b3f5 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: 96f5ab5e-ada3-44aa-a67a-a806687d5d0f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:22Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" in 23.045s (23.045s including waiting). Image size: 1334531162 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:22Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9d7ffdf6b0 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: a51a635f-cffa-4474-8319-202fafa43fea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9d890444c1 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 019a51a4-d012-4b3c-8c88-9079fd8abc8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9d8a774ecf namespace: kserve-ci-e2e-test resourceVersion: "15087" uid: 4a586af7-6ce8-45ea-876d-e67e679b9e22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9d8a8377c8 namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: d9fa0413-4054-4cc4-9d84-112733554352 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.529s (5.529s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9ed41fac12 namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: bd79f29e-ec53-4d99-b87d-ccf904b90526 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9ed8273fef namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 016d1537-d3dc-43db-9449-307fc5f5a09c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9ed8e76aba namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: 3510c1da-302d-495c-9832-2ae5ecd3ba5d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:37:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:06Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45e9f3243d014 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 5df4ffe3-80b4-4e82-87af-9f21347c838e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:48Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45eb17a5817a3 namespace: kserve-ci-e2e-test resourceVersion: "15956" uid: fa059b4d-1bc2-47f4-9def-84192e7ac588 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:48Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45eb17a5982e5 namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 6c459075-fca7-41e3-9d9c-c10451a7c5b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw namespace: kserve-ci-e2e-test resourceVersion: "14762" uid: e9f46c59-d3f7-4ece-9b53-6dcdaacc66cf kind: Event lastTimestamp: "2026-07-21T17:37:51Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:51Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw.18c45eb20e826335 namespace: kserve-ci-e2e-test resourceVersion: "16036" uid: 656de077-58c4-4bb0-b54a-5eac7b1a24b2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4dfe2-predictor-65cc4f769b namespace: kserve-ci-e2e-test resourceVersion: "14754" uid: ada9d5e0-5bc5-4b04-a08c-9bf350ff97d7 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'Created pod: error-404-isvc-4dfe2-predictor-65cc4f769b-6whlw' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: error-404-isvc-4dfe2-predictor-65cc4f769b.18c45e97db873495 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 76c2fc81-e25d-4c1b-bafc-aabd23960b72 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: bf8f0551-306a-4180-ba62-22e5a6e95f84 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Scaled up replica set error-404-isvc-4dfe2-predictor-65cc4f769b from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: error-404-isvc-4dfe2-predictor.18c45e97d94b1fce namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 4a31017a-c31d-48bd-9614-253dea6b061e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: a05c33a7-e4c3-481a-a78f-e5cee1f77c4d kind: Event lastTimestamp: "2026-07-21T17:36:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:58Z" name: error-404-isvc-4dfe2-predictor.18c45e9b57d9efad namespace: kserve-ci-e2e-test resourceVersion: "15412" uid: 8d890f90-1630-4b5f-a7f4-1ec4e37e2aa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14753" uid: a05c33a7-e4c3-481a-a78f-e5cee1f77c4d kind: Event lastTimestamp: "2026-07-21T17:36:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:58Z" name: error-404-isvc-4dfe2-predictor.18c45e9b584ed2d1 namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: 24ee5c67-4628-470c-a32b-90fc4340f1cf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:37:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: a05c33a7-e4c3-481a-a78f-e5cee1f77c4d kind: Event lastTimestamp: "2026-07-21T17:37:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:43Z" name: error-404-isvc-4dfe2-predictor.18c45ea9534d04bb namespace: kserve-ci-e2e-test resourceVersion: "15826" uid: f8c85e29-2fd3-48be-be85-78ee54813c9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:37:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15000" uid: a05c33a7-e4c3-481a-a78f-e5cee1f77c4d kind: Event lastTimestamp: "2026-07-21T17:37:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:37:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:43Z" name: error-404-isvc-4dfe2-predictor.18c45ea953cada23 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 2840c249-1378-491b-83d7-c935cac82468 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "14734" uid: 976af4f2-8f2e-454c-8c89-7507ea9b5384 kind: Event lastTimestamp: "2026-07-21T17:37:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:16Z" name: error-404-isvc-4dfe2.18c45e97d9519203 namespace: kserve-ci-e2e-test resourceVersion: "15573" uid: 8b585502-d1af-4723-8440-5672d2a87b21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: 976af4f2-8f2e-454c-8c89-7507ea9b5384 kind: Event lastTimestamp: "2026-07-21T17:37:16Z" message: InferenceService [error-404-isvc-4dfe2] is Ready metadata: creationTimestamp: "2026-07-21T17:37:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:16Z" name: error-404-isvc-4dfe2.18c45ea9edd5e7ab namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: 74ac6790-195a-481e-b472-3c19cc877116 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:37:48.789384Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:37:48Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb17fa66906 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: a0456bb8-14b2-45c7-89aa-8b582621c026 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5bf12-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb18e40a0b7 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: abdf1b34-70e3-4e58-80c6-4c070a73a3e8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1bcf46be8 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 31ca7138-7682-4261-91ee-ec7e2f6e638a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1be738e38 namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: b02a1743-7364-4597-9201-4d819eaa243b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1c1ac51bc namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 11ff8fa5-5c1d-4e5d-97ec-91ac50fd6f1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1c20e1cc1 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 47f39d86-e1c6-4235-af96-abf6d528f175 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1c2184305 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 83caa49e-b4ba-49df-9663-e7b0e19cab5e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1c4a4949e namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: d3fba331-30dd-40c6-a3c6-96b00ff29c70 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1c51e89a9 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: 433d689f-d340-4868-8cb0-4149bf212ff1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:37:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:47:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:06Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45eb1e6eee700 namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: 65c6c810-f10b-43a6-b8c7-833332d737bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45f32a8740ffa namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: f374a02c-f1d8-4a71-bb6e-91e6ed42e5cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45f32a8756f60 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 78e506cf-8df5-4deb-84ec-720f602fe4a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: ef4ef854-bdcb-419b-b4a7-ae0f4caedc69 kind: Event lastTimestamp: "2026-07-21T17:47:06Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:06Z" name: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l.18c45f335b58574c namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 783d6ba7-bd7f-41fb-a9ed-6222f98fce49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5bf12-predictor-598dcbff9d namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: 27796cf7-5b38-4d93-ab99-c948b50a269a kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'Created pod: error-404-isvc-5bf12-predictor-598dcbff9d-2wx7l' metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: error-404-isvc-5bf12-predictor-598dcbff9d.18c45eb17f218dae namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 3733bc38-4577-439f-b36e-fab3f61251f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 4ca3aa82-23d9-4b22-a67e-5b4b20c284ec kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Scaled up replica set error-404-isvc-5bf12-predictor-598dcbff9d from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: error-404-isvc-5bf12-predictor.18c45eb17d8f3104 namespace: kserve-ci-e2e-test resourceVersion: "15969" uid: 9473b3d7-bf45-4862-9ce9-41669a9cc60d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 54c7876b-cdfd-4674-a1eb-57d2035c3271 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-5bf12-predictor.18c45eb4fc172e48 namespace: kserve-ci-e2e-test resourceVersion: "16454" uid: 5fcd9534-561a-4ad7-a948-111c86f2393d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 54c7876b-cdfd-4674-a1eb-57d2035c3271 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-5bf12-predictor.18c45eb4fc7b836a namespace: kserve-ci-e2e-test resourceVersion: "16456" uid: c8e06637-41a0-401c-9255-1d0a0ab8e46d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 54c7876b-cdfd-4674-a1eb-57d2035c3271 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: error-404-isvc-5bf12-predictor.18c45ebbf9fbc976 namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 267f1b66-329e-426e-a1f5-354c4e0d5b78 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16163" uid: 54c7876b-cdfd-4674-a1eb-57d2035c3271 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: error-404-isvc-5bf12-predictor.18c45ebbfa9d3aac namespace: kserve-ci-e2e-test resourceVersion: "16920" uid: 29cd6ab9-d6b4-4910-94db-b6dadde024ba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 23923401-18ff-442c-a49c-7b5f23361218 kind: Event lastTimestamp: "2026-07-21T17:38:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:46Z" name: error-404-isvc-5bf12.18c45eb17d919f15 namespace: kserve-ci-e2e-test resourceVersion: "16734" uid: fd1999c5-2f63-4edc-aeb7-f155800b664d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16731" uid: 23923401-18ff-442c-a49c-7b5f23361218 kind: Event lastTimestamp: "2026-07-21T17:38:46Z" message: InferenceService [error-404-isvc-5bf12] is Ready metadata: creationTimestamp: "2026-07-21T17:38:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:46Z" name: error-404-isvc-5bf12.18c45ebef5f7d140 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 8a40d00c-c156-45a8-bc42-88438b866fe5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:38:18.650893Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16435" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8558d-predictor-5594879d74-cksjh to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb87388d882 namespace: kserve-ci-e2e-test resourceVersion: "16439" uid: 76df7996-ddf5-4db0-a451-369f5f4b7e8b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8558d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb87b4ff717 namespace: kserve-ci-e2e-test resourceVersion: "16453" uid: 779dcf39-3f7c-40f8-a5f0-500d4a8c8859 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8b21dd419 namespace: kserve-ci-e2e-test resourceVersion: "16475" uid: 47fa3cd3-7a53-4559-986b-2fe3753cedb5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8b393c5cc namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 33560aca-5c47-405f-b361-d5b1bee043f3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8b68c7c47 namespace: kserve-ci-e2e-test resourceVersion: "16480" uid: 628af9e0-425f-455e-9273-01ebf9ba3a1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8b72eb6bc namespace: kserve-ci-e2e-test resourceVersion: "16482" uid: d1629196-813f-4f0f-bae5-34c050240733 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8b7386800 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: d260fb76-88b3-49ba-b118-711dea7e356d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8ba605429 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 29f0906b-10a6-42ef-8737-db78fa9837bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb8bad5051f namespace: kserve-ci-e2e-test resourceVersion: "16485" uid: 64d5f084-ba5d-49ef-9dad-f255f4705ed2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:38:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:38:57Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:57Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45eb92814c89f namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 2b060436-abb8-4cb4-93ed-97fcde50c290 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45f39a4b22c0d namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: b456be37-157c-4e0b-8396-765fab2f1e02 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8558d-predictor-5594879d74-cksjh namespace: kserve-ci-e2e-test resourceVersion: "16437" uid: 0d3e5f2a-fb6e-48d6-bb0c-d7c9eaf25914 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-8558d-predictor-5594879d74-cksjh.18c45f39a4b297a2 namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: deb5565a-54f2-4ef6-a8da-e8b3651e0966 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8558d-predictor-5594879d74 namespace: kserve-ci-e2e-test resourceVersion: "16426" uid: 0d7a898a-35c5-4945-8475-1d55c5b3e8d1 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'Created pod: error-404-isvc-8558d-predictor-5594879d74-cksjh' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d-predictor-5594879d74.18c45eb8731ea387 namespace: kserve-ci-e2e-test resourceVersion: "16436" uid: d4e6db0f-7623-4be4-badf-0b2a939a78c6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16423" uid: 1ec484cc-9445-4ec4-bab7-971c662bbf61 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Scaled up replica set error-404-isvc-8558d-predictor-5594879d74 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d-predictor.18c45eb87163741d namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 35f7a50a-ea0c-4ae1-aa75-c36dd95adec0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 666bce29-580e-450e-8688-bc260391914d kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: error-404-isvc-8558d-predictor.18c45ebbf005dc96 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: 725c4299-3b5f-4d39-aa2c-36431ac62a43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16425" uid: 666bce29-580e-450e-8688-bc260391914d kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: error-404-isvc-8558d-predictor.18c45ebbf081609a namespace: kserve-ci-e2e-test resourceVersion: "16758" uid: e6372b8c-24a7-4082-9f73-751b36f2294d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 666bce29-580e-450e-8688-bc260391914d kind: Event lastTimestamp: "2026-07-21T17:39:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:33Z" name: error-404-isvc-8558d-predictor.18c45ec2ee0a10ef namespace: kserve-ci-e2e-test resourceVersion: "17199" uid: c5829db5-3c2d-4a7a-a601-44f04989411b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16618" uid: 666bce29-580e-450e-8688-bc260391914d kind: Event lastTimestamp: "2026-07-21T17:39:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:33Z" name: error-404-isvc-8558d-predictor.18c45ec2ee7494a1 namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: b7b48077-5234-4b75-b2ce-78de1be847a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: 44adfb1b-d927-431f-9e72-e06f21361bb1 kind: Event lastTimestamp: "2026-07-21T17:39:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:39:07Z" name: error-404-isvc-8558d.18c45eb8716a34c7 namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 8758dcee-c78f-46d6-a7cf-22b19949a50c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: 44adfb1b-d927-431f-9e72-e06f21361bb1 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'Failed to update status for InferenceService "error-404-isvc-8558d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8558d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d.18c45eb871e5ce27 namespace: kserve-ci-e2e-test resourceVersion: "16431" uid: ab5ceec0-920e-4ea0-a3e6-73b354d27d4f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: 44adfb1b-d927-431f-9e72-e06f21361bb1 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8558d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:18Z" name: error-404-isvc-8558d.18c45eb871e62555 namespace: kserve-ci-e2e-test resourceVersion: "16432" uid: eda64250-cd34-48fa-af2b-7297e265b860 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16962" uid: 44adfb1b-d927-431f-9e72-e06f21361bb1 kind: Event lastTimestamp: "2026-07-21T17:39:07Z" message: InferenceService [error-404-isvc-8558d] is Ready metadata: creationTimestamp: "2026-07-21T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:39:07Z" name: error-404-isvc-8558d.18c45ec3e3737a6c namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: 2b445017-4df1-475f-810d-f33e8a47ae98 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:43.938594Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21696" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8e991-predictor-5749cbc5db-9qmzg to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a0995f660 namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 5a266e49-d2a9-4ff0-a882-22514a31264f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a240162e9 namespace: kserve-ci-e2e-test resourceVersion: "21726" uid: 9144c716-4e76-47bc-b034-16f8f8b14ff7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a254c8f35 namespace: kserve-ci-e2e-test resourceVersion: "21728" uid: 590299d3-96b5-4b01-a3f8-edca193b7315 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a2779be6f namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 5a7933ac-e8c3-443a-a3d9-0c28b2f31f08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a27f0cde9 namespace: kserve-ci-e2e-test resourceVersion: "21732" uid: 28858f49-d819-4ffb-9c33-04334256da2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a280064d4 namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 94f8b6ea-88e7-4744-9a3e-ec9361a2672f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a2b1b92ab namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: bc7362bb-a05f-49e8-88d1-4a9c4677bfa7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a2b95d2e8 namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 8582e7bd-ad11-4341-b3d3-e8ed2ae83bf2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:58:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:02Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45f4a8e7b5b2c namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: 2f310358-c8ec-4432-aa2b-c8112f11a218 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45fcb30b850b4 namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 943fa7bc-5489-4928-88f3-a8a01ea6a606 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45fcb30b96c8b namespace: kserve-ci-e2e-test resourceVersion: "26247" uid: 6c5cdb83-6a95-4004-8947-b93cd46d587a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: d7a5c652-026f-4a13-9316-fde065088fb1 kind: Event lastTimestamp: "2026-07-21T17:58:02Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:02Z" name: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg.18c45fcc02d8281a namespace: kserve-ci-e2e-test resourceVersion: "26359" uid: ea422173-8a99-4293-b0ee-77782fff55cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8e991-predictor-5749cbc5db namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: c5d40553-4124-4ecf-9da5-46ad853b4c4f kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'Created pod: error-404-isvc-8e991-predictor-5749cbc5db-9qmzg' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-8e991-predictor-5749cbc5db.18c45f4a092d1c88 namespace: kserve-ci-e2e-test resourceVersion: "21697" uid: 60c29dff-2b72-42ae-8a1a-5f6db0320196 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: a7ef1896-caed-4248-93e3-b33b68e43c76 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Scaled up replica set error-404-isvc-8e991-predictor-5749cbc5db from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-8e991-predictor.18c45f4a079fa3df namespace: kserve-ci-e2e-test resourceVersion: "21687" uid: 8c96752a-0a76-4e14-8e17-a70d2a456223 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:48:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 7dd0b4f0-4398-4e77-8cce-6dcd4080d0a1 kind: Event lastTimestamp: "2026-07-21T17:49:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:28Z" name: error-404-isvc-8e991-predictor.18c45f4d86251028 namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: d4add645-e14d-41ec-86d5-aff4e15640a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:48:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 7dd0b4f0-4398-4e77-8cce-6dcd4080d0a1 kind: Event lastTimestamp: "2026-07-21T17:49:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:28Z" name: error-404-isvc-8e991-predictor.18c45f4d868b9f8e namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: de13322a-4251-4433-be6f-d0ee65188ecd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:49:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 7dd0b4f0-4398-4e77-8cce-6dcd4080d0a1 kind: Event lastTimestamp: "2026-07-21T17:50:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:13Z" name: error-404-isvc-8e991-predictor.18c45f5802a35732 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: 7928db4f-71f7-475c-a547-7fc3a30835c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:49:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 7dd0b4f0-4398-4e77-8cce-6dcd4080d0a1 kind: Event lastTimestamp: "2026-07-21T17:50:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:13Z" name: error-404-isvc-8e991-predictor.18c45f58030a0d77 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: 8af6dfe2-03e0-4e7b-86d4-5c1b769613b9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 611a1633-6d0c-490e-950a-40a1bcbac666 kind: Event lastTimestamp: "2026-07-21T17:49:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:32Z" name: error-404-isvc-8e991.18c45f4a079c25c6 namespace: kserve-ci-e2e-test resourceVersion: "22200" uid: 7990074a-dd47-42a6-97a8-434bfd8f1144 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 611a1633-6d0c-490e-950a-40a1bcbac666 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-8e991": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8e991": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-8e991.18c45f4a07edc68e namespace: kserve-ci-e2e-test resourceVersion: "21690" uid: 4c425cac-5463-49f6-9302-3fdb5f3653a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21664" uid: 611a1633-6d0c-490e-950a-40a1bcbac666 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8e991": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-8e991.18c45f4a07ee0efc namespace: kserve-ci-e2e-test resourceVersion: "21692" uid: c5e99e21-ad66-49ab-ad29-96b6ab1a177a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22197" uid: 611a1633-6d0c-490e-950a-40a1bcbac666 kind: Event lastTimestamp: "2026-07-21T17:49:32Z" message: InferenceService [error-404-isvc-8e991] is Ready metadata: creationTimestamp: "2026-07-21T17:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:32Z" name: error-404-isvc-8e991.18c45f554a3f3f60 namespace: kserve-ci-e2e-test resourceVersion: "22198" uid: baf5b4d9-7430-41df-8571-47a33b50942a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:57:58.786150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb3913ca4b namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 5440b833-0940-467c-be95-2842c94b9a3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb52e2ecc3 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 1fd3fae4-a26b-4372-a17f-73963bea0f0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb542cffa0 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: feda0995-5adf-46b7-9475-9075890aa52e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb57b091bc namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: 6c31a633-d276-48e3-a3dd-8b755f958d9a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb5822fdfc namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 8c2fd169-8f6b-4a7a-a0d2-858007c12406 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb582c2b02 namespace: kserve-ci-e2e-test resourceVersion: "26314" uid: 908eca0b-fe35-4f4b-9feb-0b3b511291e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb5b5f6ca4 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: 1209dee4-9540-4869-b4a1-b94b1f7a7558 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb5bd5fba9 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: 1bda5b31-4a01-490a-b679-9f3ee3c7c233 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T17:58:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:36Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c45fcb9258cece namespace: kserve-ci-e2e-test resourceVersion: "26666" uid: f75e3371-2702-44e7-9032-8355169d37c1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T18:15:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T18:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:18Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c460bd420ec0bc namespace: kserve-ci-e2e-test resourceVersion: "33997" uid: da82836c-ed09-4e8d-a751-453fde55b0e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T18:15:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T18:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:18Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c460bd420f4030 namespace: kserve-ci-e2e-test resourceVersion: "33998" uid: 599cba95-cc36-4124-a4c7-cbdeeba6a076 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 724ce6ed-3557-4232-9b1c-1035c5c14be7 kind: Event lastTimestamp: "2026-07-21T18:15:21Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T18:15:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:21Z" name: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h.18c460bdf2f5b5b6 namespace: kserve-ci-e2e-test resourceVersion: "34058" uid: 82a54538-6a6f-48e2-86ff-8db4b7fb66f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-984cf-predictor-58f8b9cf74 namespace: kserve-ci-e2e-test resourceVersion: "26275" uid: 47d64f52-0e59-4810-95c6-084d25ceb1f1 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'Created pod: error-404-isvc-984cf-predictor-58f8b9cf74-xkd4h' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-984cf-predictor-58f8b9cf74.18c45fcb38aae8f1 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: f5348043-f2b5-4167-af3f-4e2940356d18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: c81a61e4-4f5b-4a79-8ac6-bee3d6741e6b kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Scaled up replica set error-404-isvc-984cf-predictor-58f8b9cf74 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-984cf-predictor.18c45fcb370f6c37 namespace: kserve-ci-e2e-test resourceVersion: "26277" uid: d6468ad1-1430-49fe-b3bc-a74708fb68c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:58:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: 3d22c7d1-59ea-46cc-8903-209957d6dccd kind: Event lastTimestamp: "2026-07-21T17:58:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:43Z" name: error-404-isvc-984cf-predictor.18c45fceb593c701 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: d2b4a70d-ff32-4c97-baec-64103887d60e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:58:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: 3d22c7d1-59ea-46cc-8903-209957d6dccd kind: Event lastTimestamp: "2026-07-21T17:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:43Z" name: error-404-isvc-984cf-predictor.18c45fceb5fec1e5 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: c70da75e-78ee-4226-a31b-8f13f04f4df7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 3d22c7d1-59ea-46cc-8903-209957d6dccd kind: Event lastTimestamp: "2026-07-21T17:59:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:13Z" name: error-404-isvc-984cf-predictor.18c45fd234b7aa97 namespace: kserve-ci-e2e-test resourceVersion: "27205" uid: 1af7d0c5-8073-447e-b622-31da8c615577 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 3d22c7d1-59ea-46cc-8903-209957d6dccd kind: Event lastTimestamp: "2026-07-21T17:59:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:13Z" name: error-404-isvc-984cf-predictor.18c45fd235086f7e namespace: kserve-ci-e2e-test resourceVersion: "27206" uid: e9616016-03ca-4710-b69a-cc651f97b99e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: dd4c468c-267b-4318-b71a-4af855c72129 kind: Event lastTimestamp: "2026-07-21T17:58:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:46Z" name: error-404-isvc-984cf.18c45fcb37105b45 namespace: kserve-ci-e2e-test resourceVersion: "26786" uid: b0220aab-8972-4915-84c2-05ed2d2f743b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: dd4c468c-267b-4318-b71a-4af855c72129 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-984cf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-984cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-984cf.18c45fcb376b6080 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 196f482f-a652-4c6a-b697-715089a524bd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: dd4c468c-267b-4318-b71a-4af855c72129 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-984cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:58Z" name: error-404-isvc-984cf.18c45fcb376b9f30 namespace: kserve-ci-e2e-test resourceVersion: "26280" uid: 4294fc3e-79a5-4807-8a0e-d7723e4f5996 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: dd4c468c-267b-4318-b71a-4af855c72129 kind: Event lastTimestamp: "2026-07-21T17:58:46Z" message: InferenceService [error-404-isvc-984cf] is Ready metadata: creationTimestamp: "2026-07-21T17:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:46Z" name: error-404-isvc-984cf.18c45fd64d66ad7d namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: bb116af8-5325-499a-99ab-05b942e3f702 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:58:49.252779Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd6f91f6b54 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 8ea06306-2725-4a60-8982-1f24da5ed946 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-afaec-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd703f0b757 namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: ca53a7d5-be92-4951-9527-86a9ca47d69f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd7374a2579 namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: f55704e5-17f1-441d-a6e0-12e2f5bc3567 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd738cbcda3 namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: c8a28554-919d-4234-bf7a-2b3a2f54849f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd73af86af3 namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 907a44c3-a137-4c37-ab82-6cf63facf1aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd73b7e21fe namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: a6a50d19-bcab-4141-80cd-4cf8aa63ce28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd73b877145 namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: 6ec3e9d3-f03b-4dea-9739-08b1302e1deb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd73e70bb45 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: eb6402d3-7f35-4a2d-80cf-806b6b08eb73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd73ee16976 namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: c42ff96c-399e-42de-9ea0-a16ae4514117 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T18:08:06Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:06Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c45fd742dbed03 namespace: kserve-ci-e2e-test resourceVersion: "31011" uid: 2aabb7a6-d386-436a-9ae5-35ad3cd52c05 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T18:08:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T18:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:03Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c4605821fd59ab namespace: kserve-ci-e2e-test resourceVersion: "30988" uid: 47a70b83-ed7a-48b1-87e5-bb130d7a85c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T18:08:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T18:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:03Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c4605821fe120c namespace: kserve-ci-e2e-test resourceVersion: "30990" uid: 4d715323-e4a2-406b-a2fd-e735c9aff34f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: fc170a90-69da-4897-a7d7-6d176fdd36ba kind: Event lastTimestamp: "2026-07-21T18:08:06Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T18:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:06Z" name: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl.18c46058b73c19aa namespace: kserve-ci-e2e-test resourceVersion: "31010" uid: 53e7f761-5cc2-4a54-8005-452fa3804719 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-afaec-predictor-6f6ddbd654 namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: 77a0197c-7ef9-4fcc-8a0c-51ec12dcdc52 kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'Created pod: error-404-isvc-afaec-predictor-6f6ddbd654-jjvnl' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec-predictor-6f6ddbd654.18c45fd6f8ebcbcf namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 38bde680-5579-4389-9ae5-32be09272e9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: be8bc431-80b2-4e81-a3ce-f9d7c6abdd33 kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Scaled up replica set error-404-isvc-afaec-predictor-6f6ddbd654 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec-predictor.18c45fd6f740dae0 namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: 99042c5e-5f0f-4f08-9e7a-f03bb5bf91a8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: eaeee964-c5cf-4065-956b-5559980e55e3 kind: Event lastTimestamp: "2026-07-21T17:59:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:19Z" name: error-404-isvc-afaec-predictor.18c45fda75899ee8 namespace: kserve-ci-e2e-test resourceVersion: "27254" uid: eb1cce05-80a0-430c-8681-acdd9fcae68c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: eaeee964-c5cf-4065-956b-5559980e55e3 kind: Event lastTimestamp: "2026-07-21T17:59:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:19Z" name: error-404-isvc-afaec-predictor.18c45fda75ec282d namespace: kserve-ci-e2e-test resourceVersion: "27255" uid: 1fcce02d-6b3e-47df-8bc0-7d3e4f38eaee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:59:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: eaeee964-c5cf-4065-956b-5559980e55e3 kind: Event lastTimestamp: "2026-07-21T18:00:04Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T18:00:04Z" name: error-404-isvc-afaec-predictor.18c45fe173818211 namespace: kserve-ci-e2e-test resourceVersion: "27636" uid: 0fbdef3e-66ae-49ec-8db5-9b15d36c5a19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:59:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: eaeee964-c5cf-4065-956b-5559980e55e3 kind: Event lastTimestamp: "2026-07-21T18:00:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T18:00:04Z" name: error-404-isvc-afaec-predictor.18c45fe173ccf119 namespace: kserve-ci-e2e-test resourceVersion: "27637" uid: 6cacf7ed-363a-49cf-ac42-581ab688cd7a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: e78edffb-8f91-47ed-b421-614cd0fba54c kind: Event lastTimestamp: "2026-07-21T17:59:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:59:36Z" name: error-404-isvc-afaec.18c45fd6f71548db namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: cecc976a-d182-47b1-baa7-be42640d52ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: e78edffb-8f91-47ed-b421-614cd0fba54c kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'Failed to update status for InferenceService "error-404-isvc-afaec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-afaec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec.18c45fd6f768fb7d namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: d8c9033b-339a-4565-adfc-e16ff3c1dd62 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: e78edffb-8f91-47ed-b421-614cd0fba54c kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-afaec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-afaec.18c45fd6f769567d namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: dd24e262-a59c-4739-ac7a-5d2b554251be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: e78edffb-8f91-47ed-b421-614cd0fba54c kind: Event lastTimestamp: "2026-07-21T17:59:36Z" message: InferenceService [error-404-isvc-afaec] is Ready metadata: creationTimestamp: "2026-07-21T17:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:59:36Z" name: error-404-isvc-afaec.18c45fe1fe8af038 namespace: kserve-ci-e2e-test resourceVersion: "27394" uid: b25533e2-127c-42bc-a3a0-be081a5b6b58 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:33.639567Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b64c9-predictor-f779895b6-4rwsl to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39ab6ffb1c namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 7d85f9e7-dcd0-463d-b790-a3972182d985 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b64c9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39b73f08a5 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 314ba5c8-43cc-46f1-a362-a9966fdaae3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39e9ba2ea8 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 7e4891c7-5610-4dcb-b7b2-fec2f6b15d4d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39eb09aeb5 namespace: kserve-ci-e2e-test resourceVersion: "20757" uid: 94757e2e-a743-4d65-a0b3-349ee227bc6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39ed321cea namespace: kserve-ci-e2e-test resourceVersion: "20760" uid: 62dec538-4a18-4917-a080-22ff18591626 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39edb055c0 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 019629fe-23ba-450c-82d3-5d024dfbb57e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39edc142e5 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: d2eb6b1c-4975-46d7-bea1-5daf8caf10b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39f0913bd6 namespace: kserve-ci-e2e-test resourceVersion: "20763" uid: 17c05509-8cc1-4a64-b868-206085600fe5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f39f103ad6b namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: c885020c-df12-4aa0-ae7c-736d7a248000 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:48:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:11Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f3a2ff64cd5 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 86e6094c-43bd-4c47-a248-f9f55b5aa861 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f4a032c409e namespace: kserve-ci-e2e-test resourceVersion: "21661" uid: c5ce95ff-2dd6-4283-ab74-bf314e7479c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:43Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f4a032cc205 namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 7eb880a2-5fc1-4892-9f95-4a6ba98294fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 26970d94-f960-4da3-a5fe-adfa5d370484 kind: Event lastTimestamp: "2026-07-21T17:48:46Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:46Z" name: error-404-isvc-b64c9-predictor-f779895b6-4rwsl.18c45f4ab81cf92f namespace: kserve-ci-e2e-test resourceVersion: "21761" uid: f7dd9421-81e1-43e7-b606-83ca28d59aa6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b64c9-predictor-f779895b6 namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: fd710204-6f02-4a96-8ec2-d75bb526f82b kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'Created pod: error-404-isvc-b64c9-predictor-f779895b6-4rwsl' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9-predictor-f779895b6.18c45f39ab16138e namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 913d7a54-7dec-4993-8b36-917c5bded70a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: ffa66de8-155c-48b9-9b4c-66a5eed1833e kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Scaled up replica set error-404-isvc-b64c9-predictor-f779895b6 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9-predictor.18c45f39a96dddde namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: d82d927e-aaed-4bf8-8da1-94d7e8619dae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 8b483068-9557-45de-8e91-2fc2d8a13203 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: error-404-isvc-b64c9-predictor.18c45f3d27ace624 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 61df286a-ed14-40e3-902a-ec7b084f16f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: 8b483068-9557-45de-8e91-2fc2d8a13203 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: error-404-isvc-b64c9-predictor.18c45f3d2820087b namespace: kserve-ci-e2e-test resourceVersion: "21057" uid: 1875d854-9865-471f-8cee-52f08648bdb4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: 8b483068-9557-45de-8e91-2fc2d8a13203 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: error-404-isvc-b64c9-predictor.18c45f442590bdd4 namespace: kserve-ci-e2e-test resourceVersion: "21503" uid: a8565fd2-be5a-4a09-b99f-385d1510529f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: 8b483068-9557-45de-8e91-2fc2d8a13203 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: error-404-isvc-b64c9-predictor.18c45f4425f9db31 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 0bf75c89-cd05-4123-acd2-1257e60ae848 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: c7c428f5-fabd-4cf7-bb76-fd2f36191667 kind: Event lastTimestamp: "2026-07-21T17:48:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:22Z" name: error-404-isvc-b64c9.18c45f39a966e4e0 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 290b2b08-9e34-4a30-805b-7648712dc153 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: c7c428f5-fabd-4cf7-bb76-fd2f36191667 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'Failed to update status for InferenceService "error-404-isvc-b64c9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b64c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9.18c45f39a9ed7d30 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 444d82b9-0983-47c6-b236-859a1d22aa5c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: c7c428f5-fabd-4cf7-bb76-fd2f36191667 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b64c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-b64c9.18c45f39a9edb049 namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: ebdc2bc8-5213-4432-b877-b2aee497e1fd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: c7c428f5-fabd-4cf7-bb76-fd2f36191667 kind: Event lastTimestamp: "2026-07-21T17:48:21Z" message: InferenceService [error-404-isvc-b64c9] is Ready metadata: creationTimestamp: "2026-07-21T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:21Z" name: error-404-isvc-b64c9.18c45f44ed812cfd namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: b5c13e03-cc6b-4932-aa60-fa1d09b3d4bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:57:38.911035Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25979" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-edfd2-predictor-84887cddcb-x27z7 to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6986d9203 namespace: kserve-ci-e2e-test resourceVersion: "25986" uid: 273e83e7-e75f-4406-90de-51746e541f6d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-edfd2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6a20f612a namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 48755d02-dd8e-4cfa-b344-9aad724fede5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:57:39Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6d655bbcf namespace: kserve-ci-e2e-test resourceVersion: "26015" uid: f48f22ad-790a-4324-b707-ea71202e0e6f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6d7b1c750 namespace: kserve-ci-e2e-test resourceVersion: "26018" uid: b8a73227-67a5-4e68-84ea-a462aeb6622b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6da95eaff namespace: kserve-ci-e2e-test resourceVersion: "26021" uid: 168e144b-22f4-4a79-9224-f236ff6d665b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6db12a594 namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: 9d0f5ff7-0a52-4c25-9bdf-2ef61a238af8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6db1beee4 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 886390ac-5fdb-43ab-ac4a-dd1e0dc4557d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6dddfb399 namespace: kserve-ci-e2e-test resourceVersion: "26028" uid: ddd4c722-06b1-4ed9-954c-85b2018b3f52 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc6de5b04d4 namespace: kserve-ci-e2e-test resourceVersion: "26029" uid: c032e5a8-b6e4-4ba3-83fa-1769cbec96cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:58:17Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:17Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fc7206af0fd namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: 9c940f4b-3f72-460f-80f9-bc675704dd38 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fd6f2710925 namespace: kserve-ci-e2e-test resourceVersion: "26881" uid: fe7963d2-5a2d-4857-a98f-576cbfa43395 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fd6f2717df3 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 12e41d8d-70a1-4239-83b4-f1e10e718d49 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7 namespace: kserve-ci-e2e-test resourceVersion: "25983" uid: 06da2dde-96ec-4bbe-9eac-69de21849e6a kind: Event lastTimestamp: "2026-07-21T17:58:52Z" message: 'Readiness probe failed: Get "https://10.133.0.36:8643/healthz": dial tcp 10.133.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:52Z" name: error-404-isvc-edfd2-predictor-84887cddcb-x27z7.18c45fd7a8bc1165 namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: cd6adc49-f55d-4edd-83e2-f00528877043 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-edfd2-predictor-84887cddcb namespace: kserve-ci-e2e-test resourceVersion: "25970" uid: 8cc63358-cb57-4c3d-9b53-d285c246ce86 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'Created pod: error-404-isvc-edfd2-predictor-84887cddcb-x27z7' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-edfd2-predictor-84887cddcb.18c45fc697de5bcf namespace: kserve-ci-e2e-test resourceVersion: "25984" uid: 54f60c77-471f-41ae-ad1e-a37357c653cd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25962" uid: 143fb556-b83c-4c08-a602-8201585a5630 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Scaled up replica set error-404-isvc-edfd2-predictor-84887cddcb from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-edfd2-predictor.18c45fc695ff33a7 namespace: kserve-ci-e2e-test resourceVersion: "25973" uid: 973249e7-a121-4881-a991-1bb64a82ffb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:57:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 116b6b25-7b76-45d3-a00c-c8b45dfb22a8 kind: Event lastTimestamp: "2026-07-21T17:58:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:23Z" name: error-404-isvc-edfd2-predictor.18c45fca14b3baff namespace: kserve-ci-e2e-test resourceVersion: "26549" uid: cd8a255b-6da4-490c-b286-539abdcee919 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:57:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 116b6b25-7b76-45d3-a00c-c8b45dfb22a8 kind: Event lastTimestamp: "2026-07-21T17:58:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:23Z" name: error-404-isvc-edfd2-predictor.18c45fca14ff5cb4 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 83dbca9a-34a6-48e7-ac9e-82fc7bc42ee2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 116b6b25-7b76-45d3-a00c-c8b45dfb22a8 kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: error-404-isvc-edfd2-predictor.18c45fd4910f94b2 namespace: kserve-ci-e2e-test resourceVersion: "26705" uid: 68a4621c-3ed6-409e-9840-e5894a21d5c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 116b6b25-7b76-45d3-a00c-c8b45dfb22a8 kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: error-404-isvc-edfd2-predictor.18c45fd49198ecff namespace: kserve-ci-e2e-test resourceVersion: "26708" uid: 45787783-ff68-4c1a-bd97-495db99f5791 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 707bc77c-d9fd-4542-be85-72a7f6e5029a kind: Event lastTimestamp: "2026-07-21T17:58:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:27Z" name: error-404-isvc-edfd2.18c45fc69698c2d9 namespace: kserve-ci-e2e-test resourceVersion: "26595" uid: b4ba30c6-6258-4bd7-981a-639276427655 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 707bc77c-d9fd-4542-be85-72a7f6e5029a kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-edfd2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-edfd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-edfd2.18c45fc696fa7ea5 namespace: kserve-ci-e2e-test resourceVersion: "25977" uid: 3f4dea5f-a1d2-4586-8703-0565c7146ee2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25940" uid: 707bc77c-d9fd-4542-be85-72a7f6e5029a kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-edfd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:38Z" name: error-404-isvc-edfd2.18c45fc696fada32 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 274f07ab-40b1-4b82-a4a2-486d49cdb37f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "26592" uid: 707bc77c-d9fd-4542-be85-72a7f6e5029a kind: Event lastTimestamp: "2026-07-21T17:58:27Z" message: InferenceService [error-404-isvc-edfd2] is Ready metadata: creationTimestamp: "2026-07-21T17:58:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:27Z" name: error-404-isvc-edfd2.18c45fd1dbfb4a18 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: 3e893168-47d3-4fd4-b901-56847d0965c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:03.609843Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32ad86c520 namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: 1e227666-5327-4e41-9a73-d8f42c48f91b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f64bd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32b7487605 namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: d9eaabc4-350f-49f1-b381-1429f5c17323 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20342" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32eb9602ac namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 18e584d8-1322-4706-bc73-0bb3544c408a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32ecf047b2 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 3adf28a6-2403-4f4f-903e-f8a1fa6d6e13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32f04761f0 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: b5149cf9-2fd1-4b2e-a8c9-c305d0f2df73 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32f0c72d4e namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 214de8ff-842e-42b5-93aa-c5881d235181 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32f0d1b9b7 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 9d11a533-8e53-4999-8a6a-3d85a2855b2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32f4128c79 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 2bf7540d-5366-4ea9-a381-5dbe9c5202d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f32f4912336 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 9c1c7eeb-c9b7-4895-864b-316aa4c58113 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:47:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:41Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f332bc49da7 namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 6f6983f9-a184-4fd9-8da6-730125ca1323 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f42ffd0cc6a namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: fbe38ae0-6dce-4da1-81f9-b09803b26bef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:13Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f42ffd1ad66 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: b6ec4a15-20bb-4e43-9d60-b7deb63e1ccf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7 namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 26f7402a-85bc-48c6-b8aa-2d192f06bd3c kind: Event lastTimestamp: "2026-07-21T17:48:16Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:16Z" name: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7.18c45f43b42bb004 namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 4f53cd6b-e9f5-46ef-a31e-436b60ba73f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f64bd-predictor-76c89fcd66 namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 62038993-55a6-477a-a24d-0d79900e0a82 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: 'Created pod: error-404-isvc-f64bd-predictor-76c89fcd66-2mzm7' metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-f64bd-predictor-76c89fcd66.18c45f32acf185d5 namespace: kserve-ci-e2e-test resourceVersion: "20333" uid: b722cdfa-ef11-4444-bead-35a664a1bc90 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: df2b1435-a891-48c2-92bd-4f0f93f9e0a6 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Scaled up replica set error-404-isvc-f64bd-predictor-76c89fcd66 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:03Z" name: error-404-isvc-f64bd-predictor.18c45f32ab5799dd namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: abc77dad-4713-4058-93a1-651dd67d2fe4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 737ba670-4600-43ec-aaa7-9ec7cd24c50b kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-f64bd-predictor.18c45f36298fef61 namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: d3e22890-ccd3-4f34-937c-1f44cbf3055e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 737ba670-4600-43ec-aaa7-9ec7cd24c50b kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: error-404-isvc-f64bd-predictor.18c45f3629fa264e namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 836c15c7-a2c3-499c-97df-75d3cb77fc96 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 737ba670-4600-43ec-aaa7-9ec7cd24c50b kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: error-404-isvc-f64bd-predictor.18c45f3d278c6fb9 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: cd6babcf-13b6-4bd1-abdc-593214934da8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 737ba670-4600-43ec-aaa7-9ec7cd24c50b kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: error-404-isvc-f64bd-predictor.18c45f3d27ef8eee namespace: kserve-ci-e2e-test resourceVersion: "21053" uid: b9c38e19-95bc-4b3c-b47a-9d933d5d87f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64bd namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: cf55d24b-3da6-48ce-bd86-d2a4f95be07a kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:51Z" name: error-404-isvc-f64bd.18c45f32ab2b1ddb namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: f72b9446-6bfb-42c2-aee5-921e8c242f86 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f64bd namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: cf55d24b-3da6-48ce-bd86-d2a4f95be07a kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: InferenceService [error-404-isvc-f64bd] is Ready metadata: creationTimestamp: "2026-07-21T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:51Z" name: error-404-isvc-f64bd.18c45f3de994761b namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 2ecfc443-8c59-47a5-96a0-c839fc0db616 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:58.578486Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e97d691a272 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 2b18add8-94ac-468f-bfee-6eca797ece20 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14752" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e97f0347f0e namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: e99d936a-a748-40fa-a2bc-e2e1f32d7d71 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e97f225dfb7 namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 9628cda0-ec17-47c0-8c4d-ce1d905d5a15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e97f5153708 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 7fca3298-4a61-4081-ab0f-b85ac6399276 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e97f588f24a namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: daecb95e-263f-42fe-a290-b0683414e911 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:03Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" metadata: creationTimestamp: "2026-07-21T17:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:03Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9918da3810 namespace: kserve-ci-e2e-test resourceVersion: "14916" uid: 8e99e6c2-028b-4b3f-8f55-1a514a8c0436 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" in 18.909s (18.909s including waiting). Image size: 1558090149 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:22Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9d7ff1f4e8 namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: bc2a5e6b-74b3-4baa-9e5b-e9811fdbc1f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9d87aac87b namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 8a46a3dd-ea60-4439-916f-88fbc432b660 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9d899c78e2 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: b6507f40-13ca-410b-9f3a-022375849abe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9d89a831d4 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 03aaf58f-62d5-4d9e-a642-5da3b3d1d50d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.482s (5.482s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9ed0707efc namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 4d5190a5-21f5-4a56-bec1-80852a375210 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9ed82619da namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 4f26d440-fc12-4e6d-a018-db0c0b806c49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9ed8e608fc namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: f92a318a-22ec-4278-8235-40462d2d7506 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:37:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:36Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45e9f32402fb3 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: f719e535-68f5-4ff6-ad42-dd613ae8cd7c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45eb86b4148ac namespace: kserve-ci-e2e-test resourceVersion: "16382" uid: f2692581-00f7-49dc-8899-ea66c689e21a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45eb86b41bc70 namespace: kserve-ci-e2e-test resourceVersion: "16387" uid: ac2a8338-9371-44d1-99b6-0c404dfca52f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 69e0a43b-e6af-4859-9a8f-872ada531c9d kind: Event lastTimestamp: "2026-07-21T17:38:21Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:21Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt.18c45eb90aad4de3 namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: 19ac0843-e75d-4964-b1e7-1aed56b951b0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84695d5b5c namespace: kserve-ci-e2e-test resourceVersion: "14727" uid: 1531baa0-2bca-467e-a9e5-86cc7e03ea53 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84695d5b5c-x96vt' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-1-predictor-84695d5b5c.18c45e97d61ffb62 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 51c42817-bc36-4aa5-bb19-3ed43560a066 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14724" uid: 1f56583e-d54c-4257-850a-37bb0a6d3f2c kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84695d5b5c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-1-predictor.18c45e97d009b22c namespace: kserve-ci-e2e-test resourceVersion: "14731" uid: a302deba-029c-4e82-9ed4-1e31924fd4e8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: a82d70d2-92ac-4cd8-82ce-42cd724b701f kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: isvc-sklearn-graph-1-predictor.18c45e9b4e34a4c8 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: f9944bcf-b29f-40ec-abe4-479da794a9b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14725" uid: a82d70d2-92ac-4cd8-82ce-42cd724b701f kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: isvc-sklearn-graph-1-predictor.18c45e9b4ea56743 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: 709fe336-3c25-40f9-94e3-886d577dd475 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: a82d70d2-92ac-4cd8-82ce-42cd724b701f kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-sklearn-graph-1-predictor.18c45ea5cb0fd1c4 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 3536e877-5552-4d2c-9205-b34acad2071c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: a82d70d2-92ac-4cd8-82ce-42cd724b701f kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-sklearn-graph-1-predictor.18c45ea5cb7b1667 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: bf0add2b-79bd-4c8b-a394-a235bb1d5de6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 2faa2a88-3970-4047-87b4-55ef28c08131 kind: Event lastTimestamp: "2026-07-21T17:37:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:46Z" name: isvc-sklearn-graph-1.18c45e97d00b7b96 namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: 3b882460-55bb-46bb-a48c-3a7841981c98 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 2faa2a88-3970-4047-87b4-55ef28c08131 kind: Event lastTimestamp: "2026-07-21T17:37:46Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-21T17:37:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:46Z" name: isvc-sklearn-graph-1.18c45eb0e9d230b1 namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: f15b4e4b-7070-4f23-a5db-cb29515c6f0a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:58.897808Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-868d77456c-snz9c to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e97e999fc0d namespace: kserve-ci-e2e-test resourceVersion: "14821" uid: 8af70c33-be9a-4d1e-9898-cd329174c0a8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e98044110a9 namespace: kserve-ci-e2e-test resourceVersion: "14854" uid: 612d9e3d-6896-4af9-b3e7-8c620c0ef339 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e98062357e3 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 724f1b53-1c21-4294-9174-ff3005b9e39e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e980a142cdf namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 9fd9c8b4-1844-4d75-825c-cb3473caa7a6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e980ae4ab5f namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 84439b93-8ca6-4273-b856-869f572982a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:03Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" metadata: creationTimestamp: "2026-07-21T17:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:03Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e991923a48e namespace: kserve-ci-e2e-test resourceVersion: "14917" uid: da1d86a9-7521-4bb1-8b6a-acedd00dfb64 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:22Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1775" in 18.952s (18.952s including waiting). Image size: 1558090149 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:22Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9d82ccaabb namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 5ea6309f-eea2-4a43-bd13-6e31459636b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9d8af143a7 namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: ba0b9593-7065-489d-b0d8-1181cd263a60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9d8c08cc68 namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: 095a1ca2-13f3-4afd-b87d-8035efdc8a44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9d8c15ced8 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 9fe0770d-ea60-43c0-8641-f5a75d621601 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.487s (5.487s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9ed32b8ad1 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 85848ad4-6574-475a-b454-8d753069c954 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9ed822472f namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: 0550ed56-5aeb-4efe-9526-420bb124f2f7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9ed8ab10a8 namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: c864eb2d-0f37-49c2-ae1c-8401765e45d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:37:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:43Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45e9ef607fb51 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 709f972c-a547-465f-ba0b-4284487cd8ca reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45eb8683f8186 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: d0158f0d-6661-43ee-bc8b-45c49affaea0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45eb8684006b7 namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 07b4600f-0d16-47dc-a3cd-d1508e9cf3db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:38:20Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:20Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45eb8ceb95dc9 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: 1bcbf7e3-a80c-4ba6-9de6-3fb254d5b7e6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 6e8d920d-8b7d-4536-bae9-593c6439d608 kind: Event lastTimestamp: "2026-07-21T17:38:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: i/o timeout' metadata: creationTimestamp: "2026-07-21T17:38:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:24Z" name: isvc-sklearn-graph-2-predictor-868d77456c-snz9c.18c45eb9dfa7eef4 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 6bd1888e-1173-4137-b27a-d5bba5fd24e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-868d77456c namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 16d10f3f-dca7-4b3d-a7fe-29fbc8b85801 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-868d77456c-snz9c' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-2-predictor-868d77456c.18c45e97e8fadaa5 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: efc590b0-55ff-46d2-9efa-1aaff665f2b7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: e4d0437d-e476-4566-a0cf-37e2788231fe kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-868d77456c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-sklearn-graph-2-predictor.18c45e97e6472f39 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 020cf223-46c0-40cc-8e08-e05e6ab82b73 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 0ab22a14-aea7-4885-83d0-9ff222123512 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: isvc-sklearn-graph-2-predictor.18c45e9b64d8668c namespace: kserve-ci-e2e-test resourceVersion: "15287" uid: 376f4ed2-5d48-4dcf-bfcc-2708879c8e3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 0ab22a14-aea7-4885-83d0-9ff222123512 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: isvc-sklearn-graph-2-predictor.18c45e9b653f6161 namespace: kserve-ci-e2e-test resourceVersion: "15289" uid: 288606db-2ca8-41e5-bf1f-4c51de0e3f0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 0ab22a14-aea7-4885-83d0-9ff222123512 kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-sklearn-graph-2-predictor.18c45ea5e1c23c10 namespace: kserve-ci-e2e-test resourceVersion: "16279" uid: df2d98a6-5445-4c92-8d9d-f0e3acab6549 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15008" uid: 0ab22a14-aea7-4885-83d0-9ff222123512 kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-sklearn-graph-2-predictor.18c45ea5e22c84a0 namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: cc89c41b-d79f-4c10-928d-b390c48743e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: fdacf68a-f78a-495b-8423-5ef8f0eb2f05 kind: Event lastTimestamp: "2026-07-21T17:37:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:53Z" name: isvc-sklearn-graph-2.18c45e97e64cdea0 namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: bd9fc512-6dd3-4995-b08f-fc908418fbdc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: fdacf68a-f78a-495b-8423-5ef8f0eb2f05 kind: Event lastTimestamp: "2026-07-21T17:37:53Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-21T17:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:53Z" name: isvc-sklearn-graph-2.18c45eb2ad2cffeb namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: af120343-3f17-425c-8a90-4d95599cd983 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:58.792189Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-2w9gs to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e97e34e6cf9 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: c8cfe3cb-d0e1-477f-80b8-b9d77f008b88 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e97f11f5aac namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: d8631ae4-bd07-47ae-82cd-56a6e758b9d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e9827ba430e namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 0c80a860-129e-420a-a776-c40ebad1af68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:35:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:7e12dde312cd424230e1ef8053d8890ee8a23543039767c0a577fe647c927b0a" already present on machine metadata: creationTimestamp: "2026-07-21T17:35:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e982980900f namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: bcc09c14-dd0a-4f8d-86e8-683a81098997 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-21T17:36:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:00Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e982ef330b2 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: f7c0e851-16dd-449e-9c6f-dabc4688dde6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-21T17:36:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:00Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e9830210fdb namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: 3c05e40e-ac24-47ad-8b95-9f87197995a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:07Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-21T17:36:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45e99ce321418 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 5e19f463-bb03-4f0a-bd53-014b07769f0b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 36.615s (36.615s including waiting). Image size: 1307347707 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea254a04acb namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: 197ffc19-a55c-45ef-8a77-e6a6ed24b159 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea25adbdc44 namespace: kserve-ci-e2e-test resourceVersion: "15279" uid: 98e3839f-39aa-4fe7-8122-c1ca3ed61dfc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea25c26bc4f namespace: kserve-ci-e2e-test resourceVersion: "15281" uid: 9a3ff309-f234-464e-93d6-711b2c463a64 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea25c3df3a6 namespace: kserve-ci-e2e-test resourceVersion: "15282" uid: a3621527-578c-4cb4-a2c7-a388f19a5139 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea260b9d538 namespace: kserve-ci-e2e-test resourceVersion: "15285" uid: 28b82783-329c-406d-884f-d873f4139b4c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:36:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea2622a456d namespace: kserve-ci-e2e-test resourceVersion: "15286" uid: fb4a32cf-438e-4977-af97-88010dbbb83d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-21T17:36:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:38:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:36:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45ea2b475de5d namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: a8d4e828-5c09-47aa-a65a-1e5ab63bf2e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45eb86e97397d namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: c35b3d5e-fecd-4faa-b8da-84b60df615c9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45eb86e976390 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 7230f74c-43d2-424e-aeae-0677763fb78f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: 8c5e3cb0-172b-4f1a-8e76-028bbc1c8f10 kind: Event lastTimestamp: "2026-07-21T17:38:21Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:38:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-2w9gs.18c45eb90ece41bd namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 79019d3b-aa85-447c-ab88-c6becd94219a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 572a3eb8-30dd-43a1-b5ef-7623c2065177 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-2w9gs' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c45e97e2dea19e namespace: kserve-ci-e2e-test resourceVersion: "14791" uid: a61932cc-4ddd-4f68-8f2c-98ef606a11d3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: ba7d6931-54cd-4000-8682-d0fda4fbe1fb kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: isvc-xgboost-graph-predictor.18c45e97e03b9fa6 namespace: kserve-ci-e2e-test resourceVersion: "14785" uid: 4f63a42b-b913-4449-a705-34c1bf275cd5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 4773e2c4-8f93-4b27-ade9-4c117ef9b8ca kind: Event lastTimestamp: "2026-07-21T17:37:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:28Z" name: isvc-xgboost-graph-predictor.18c45e9b5ec31aea namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: f8cdc22c-f26b-4689-8f0d-80ddca19caaa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 4773e2c4-8f93-4b27-ade9-4c117ef9b8ca kind: Event lastTimestamp: "2026-07-21T17:37:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:28Z" name: isvc-xgboost-graph-predictor.18c45e9b5f32a29f namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: ebbdc3c3-5421-4b40-86ed-8cd37c63db42 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:37:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 4773e2c4-8f93-4b27-ade9-4c117ef9b8ca kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-xgboost-graph-predictor.18c45eb0582c3329 namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: f93c029a-2949-4728-9e13-fdc5d86bf53a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:37:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15005" uid: 4773e2c4-8f93-4b27-ade9-4c117ef9b8ca kind: Event lastTimestamp: "2026-07-21T17:38:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:37:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:13Z" name: isvc-xgboost-graph-predictor.18c45eb058a5084c namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 599e54c1-398e-458e-8ac1-25f1affc0dfd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14764" uid: adab885a-0fe9-4406-b7a5-ec0374601dfd kind: Event lastTimestamp: "2026-07-21T17:37:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:51Z" name: isvc-xgboost-graph.18c45e97e0377819 namespace: kserve-ci-e2e-test resourceVersion: "16046" uid: 4c6210a2-3f6d-4cbe-a3d2-2c5f2aad4ad6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16044" uid: adab885a-0fe9-4406-b7a5-ec0374601dfd kind: Event lastTimestamp: "2026-07-21T17:37:51Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-21T17:37:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:51Z" name: isvc-xgboost-graph.18c45eb218b30632 namespace: kserve-ci-e2e-test resourceVersion: "16045" uid: 30184240-df18-4a09-b207-a14710283589 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:38:08.231311Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-b94c8c4fd-9pj2p to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb6067a9faa namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 49caafbb-ace1-416f-9e66-55e2e9de014e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb6200edf77 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: b83de826-b64f-4f4e-ab31-acfafa075f3d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb62146116a namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: e6058b40-b524-4880-9f0c-b720fc5dfb91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb6244cc77e namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: e74327f4-af3b-4344-9ae6-5ed55cd35421 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb624c02648 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: ffcfb89f-19f7-449e-a805-79a3992c64d6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb861cb97a2 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: de5aa782-3039-4e0d-8451-6a5d0eb1bb6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b94c8c4fd-9pj2p namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 4703a6d4-0761-4b4b-9bdb-2aa0e274aee8 kind: Event lastTimestamp: "2026-07-21T17:38:45Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:45Z" name: model-chainer-b94c8c4fd-9pj2p.18c45eb8d664e6a9 namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: 3e88e76e-133d-4b1d-a7d7-e62329a0bcfb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-b94c8c4fd namespace: kserve-ci-e2e-test resourceVersion: "16210" uid: d60facf2-27bc-46b2-a4dc-80b63a159cf0 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: 'Created pod: model-chainer-b94c8c4fd-9pj2p' metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer-b94c8c4fd.18c45eb6063cc942 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: f8cb2bbe-9b8d-46ed-b6de-7c3d1b87d14d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: a3d5620f-eab6-4fbf-8222-68e735f32655 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: Scaled up replica set model-chainer-b94c8c4fd from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer.18c45eb6056b6e11 namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: cbee6a9b-0ec5-439a-bea4-75e54dd1ffeb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16224" uid: 764eea94-4fe6-453c-baab-20eccb9960a2 kind: Event lastTimestamp: "2026-07-21T17:38:08Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-21T17:38:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:08Z" name: model-chainer.18c45eb60b35cf00 namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: 0f22b1d3-e3e0-408f-ae4a-4c72a76f0b14 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:39:18.568737Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17046" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8558d-55554bfd8c-dq7wz to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:39:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:39:18Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45ec666ea9894 namespace: kserve-ci-e2e-test resourceVersion: "17052" uid: ebd293b7-d43d-49b5-bbb5-76093846a38c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:18Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:39:19Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45ec68095d55e namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: f1466816-dd7c-4f70-a460-f604c94f87fb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8558d} kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:39:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:19Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45ec681ce9510 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 280b5620-b229-4c25-b469-9758a4fa74b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8558d} kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:39:19Z" message: 'Created container: sequence-graph-8558d' metadata: creationTimestamp: "2026-07-21T17:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:19Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45ec68495c08f namespace: kserve-ci-e2e-test resourceVersion: "17078" uid: 8a7fa03f-2ba4-4995-be42-088ea16abc82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8558d} kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:39:19Z" message: Started container sequence-graph-8558d metadata: creationTimestamp: "2026-07-21T17:39:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:39:19Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45ec68507c420 namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: 5ebda565-a2b0-4658-94c2-fb8394a7ef88 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8558d} kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Stopping container sequence-graph-8558d metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45f399a8e441d namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 79899587-e794-4b81-9c7f-174d45301474 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8558d} kind: Pod name: sequence-graph-8558d-55554bfd8c-dq7wz namespace: kserve-ci-e2e-test resourceVersion: "17048" uid: 6ef6ba6a-2f42-400f-9cb1-80300498c6b4 kind: Event lastTimestamp: "2026-07-21T17:48:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:00Z" name: sequence-graph-8558d-55554bfd8c-dq7wz.18c45f3a22490687 namespace: kserve-ci-e2e-test resourceVersion: "21001" uid: e84bbc95-a0bc-4200-ab5d-4a4a7c787eaa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8558d-55554bfd8c namespace: kserve-ci-e2e-test resourceVersion: "17043" uid: c9b0fdb4-3beb-4c41-b94f-a1222bff1dab kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: 'Created pod: sequence-graph-8558d-55554bfd8c-dq7wz' metadata: creationTimestamp: "2026-07-21T17:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:18Z" name: sequence-graph-8558d-55554bfd8c.18c45ec66679b9b4 namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: e3d812ca-9119-4f5e-8488-688f7de6695e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8558d namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 4023b152-c19c-41d6-9df7-8bd02dcdabdb kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: Scaled up replica set sequence-graph-8558d-55554bfd8c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:18Z" name: sequence-graph-8558d.18c45ec665b6e9a9 namespace: kserve-ci-e2e-test resourceVersion: "17044" uid: 130d41bd-b1f0-4ede-a3f4-be8764a6c761 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8558d namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: 0d6a3b03-6f94-4f12-8af9-f28db2b42965 kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: InferenceGraph [sequence-graph-8558d] is Ready metadata: creationTimestamp: "2026-07-21T17:39:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:39:18Z" name: sequence-graph-8558d.18c45ec66af2df7e namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 8dd36b1e-964a-4ec5-b48d-32152fcc4b02 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:39:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8558d namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: a9701291-c55a-44b7-93f7-176718324078 kind: Event lastTimestamp: "2026-07-21T17:39:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:48Z" name: sequence-graph-8558d.18c45ec9e4726cc7 namespace: kserve-ci-e2e-test resourceVersion: "17304" uid: 21cdeb8e-89c5-475a-827a-bd1940b0d86d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:39:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8558d namespace: kserve-ci-e2e-test resourceVersion: "17042" uid: a9701291-c55a-44b7-93f7-176718324078 kind: Event lastTimestamp: "2026-07-21T17:39:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:48Z" name: sequence-graph-8558d.18c45ec9e4e34118 namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: c95106df-48b1-4c0c-b8f0-2f185f8019f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:49:43.911693Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22299" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8e991-bb5b488f6-295l9 to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:49:43Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45f580042d8ac namespace: kserve-ci-e2e-test resourceVersion: "22304" uid: 0c63d6ac-4629-40b1-b0ce-3f439c7ab78c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:49:44Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:49:44Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45f5819d34bfb namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: b954cb6d-7b2f-4b9d-9db6-774c35947f79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8e991} kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:49:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:44Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45f581b1aee58 namespace: kserve-ci-e2e-test resourceVersion: "22327" uid: 1d16bbd7-d76b-4667-ab6b-b9c3eb889675 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8e991} kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:49:44Z" message: 'Created container: sequence-graph-8e991' metadata: creationTimestamp: "2026-07-21T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:44Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45f581e06c3ce namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: d1c4c7df-e36a-4077-af99-b643fabb3a33 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8e991} kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:49:44Z" message: Started container sequence-graph-8e991 metadata: creationTimestamp: "2026-07-21T17:49:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:44Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45f581e785e0e namespace: kserve-ci-e2e-test resourceVersion: "22330" uid: fa16b5e7-f478-41bb-9d6e-3a25efc474dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8e991} kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Stopping container sequence-graph-8e991 metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45fcb28287386 namespace: kserve-ci-e2e-test resourceVersion: "26203" uid: a053f2d3-d1df-4310-aa98-44007ea7d21a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8e991} kind: Pod name: sequence-graph-8e991-bb5b488f6-295l9 namespace: kserve-ci-e2e-test resourceVersion: "22301" uid: 1fba3291-23ba-4393-8b98-54390aad47cc kind: Event lastTimestamp: "2026-07-21T17:58:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:25Z" name: sequence-graph-8e991-bb5b488f6-295l9.18c45fcb9b689144 namespace: kserve-ci-e2e-test resourceVersion: "26561" uid: 26f01af4-d2e8-4b45-bb0b-1914ba56cff4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8e991-bb5b488f6 namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: b7c13e82-f427-4279-ba45-db7bbcc06ae1 kind: Event lastTimestamp: "2026-07-21T17:49:43Z" message: 'Created pod: sequence-graph-8e991-bb5b488f6-295l9' metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:43Z" name: sequence-graph-8e991-bb5b488f6.18c45f57ffdb1a0b namespace: kserve-ci-e2e-test resourceVersion: "22300" uid: f45217c7-3d8e-411e-ae36-6d188ee16e84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: 87fbe8c2-076d-4c3e-9daa-00c23584bf56 kind: Event lastTimestamp: "2026-07-21T17:49:43Z" message: Scaled up replica set sequence-graph-8e991-bb5b488f6 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:43Z" name: sequence-graph-8e991.18c45f57ff185fcb namespace: kserve-ci-e2e-test resourceVersion: "22297" uid: 05a8a625-1061-420e-916e-f78a626bb0ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22314" uid: 5a93dc86-6073-47fd-90bc-d13532d1b884 kind: Event lastTimestamp: "2026-07-21T17:49:43Z" message: InferenceGraph [sequence-graph-8e991] is Ready metadata: creationTimestamp: "2026-07-21T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:43Z" name: sequence-graph-8e991.18c45f5804672fa6 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: 73a630c3-4202-40ec-a5ea-f91049c96d1d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 8049fa6d-f5bd-4daa-b94e-3c39b1c84f7c kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: sequence-graph-8e991.18c45f5b7da73223 namespace: kserve-ci-e2e-test resourceVersion: "22443" uid: 3ab6e283-f152-4cb5-a9fe-95ba4e74fc69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 8049fa6d-f5bd-4daa-b94e-3c39b1c84f7c kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:49:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: sequence-graph-8e991.18c45f5b7e48e9b9 namespace: kserve-ci-e2e-test resourceVersion: "22445" uid: 10efea77-c7d3-4a0f-8974-9065c85e0c5f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: 8049fa6d-f5bd-4daa-b94e-3c39b1c84f7c kind: Event lastTimestamp: "2026-07-21T17:50:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:13Z" name: sequence-graph-8e991.18c45f5efcfe4663 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: acf70b58-0975-42b6-a6bf-b2908b28798d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:50:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22444" uid: 8049fa6d-f5bd-4daa-b94e-3c39b1c84f7c kind: Event lastTimestamp: "2026-07-21T17:50:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:50:13Z" name: sequence-graph-8e991.18c45f5efd4d173d namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: c6909c75-0aa8-4fbe-98e6-f777d4438445 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:33.559356Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-b64c9-cbbfb58f7-5whfl to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:33Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f479eef62b6 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: 52596d4b-b325-468c-93e7-4d41b87f601a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:33Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21496" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:33Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f47b8b71a9c namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 42f9feb9-334a-405c-970b-6dc655e59621 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b64c9} kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:48:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:34Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f47ba0dd2e4 namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: 3c3b1e2b-b1a4-4297-81b5-d6f872ac54e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b64c9} kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:48:34Z" message: 'Created container: sequence-graph-b64c9' metadata: creationTimestamp: "2026-07-21T17:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:34Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f47bdf2583f namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: ba5a828d-e891-4a7f-b1ad-ffd0364eb40b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b64c9} kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:48:34Z" message: Started container sequence-graph-b64c9 metadata: creationTimestamp: "2026-07-21T17:48:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:34Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f47be6b1cd0 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 03cbc89a-8326-4c96-90d0-94054bfe9449 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b64c9} kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Stopping container sequence-graph-b64c9 metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:43Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f49f9412bd6 namespace: kserve-ci-e2e-test resourceVersion: "21607" uid: 60684cda-b841-4fb7-9e6d-59bff801af57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-b64c9} kind: Pod name: sequence-graph-b64c9-cbbfb58f7-5whfl namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: b9719340-89dd-48a3-adca-417eb7fe6c9f kind: Event lastTimestamp: "2026-07-21T17:49:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:49:10Z" name: sequence-graph-b64c9-cbbfb58f7-5whfl.18c45f4a4f7f6c7f namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: b666a83a-f25e-421c-812b-78057ee029e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-b64c9-cbbfb58f7 namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 1df28912-d1cb-456e-9ef4-3ce58fc0f1e2 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: 'Created pod: sequence-graph-b64c9-cbbfb58f7-5whfl' metadata: creationTimestamp: "2026-07-21T17:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: sequence-graph-b64c9-cbbfb58f7.18c45f479e976af3 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 1fc56c56-20ee-44e3-bbbe-8b6827410408 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: dce9ca5b-144b-47fd-a3b6-6331ac469369 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: Scaled up replica set sequence-graph-b64c9-cbbfb58f7 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: sequence-graph-b64c9.18c45f479df09d15 namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: 73b35357-89ef-4a39-a749-7f89fb605fd3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "21498" uid: 19c09d09-a384-452f-b15a-585bd921e777 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: InferenceGraph [sequence-graph-b64c9] is Ready metadata: creationTimestamp: "2026-07-21T17:48:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:33Z" name: sequence-graph-b64c9.18c45f47a34e5865 namespace: kserve-ci-e2e-test resourceVersion: "21500" uid: 4ca227bb-0eee-4cea-a029-871a4def3e77 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:59:49.226706Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27502" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-afaec-6f497d467-xk7jj to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c45fe4efd8ef56 namespace: kserve-ci-e2e-test resourceVersion: "27507" uid: f9b7520e-cef2-4913-8fd1-f590f8a1744d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c45fe509dbbd13 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 46c7db71-6416-4975-8d9d-a264c4724ea8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afaec} kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c45fe50b239e4e namespace: kserve-ci-e2e-test resourceVersion: "27526" uid: ddf20b3f-a695-456e-9267-4df816bcfa60 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afaec} kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: 'Created container: splitter-graph-afaec' metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c45fe50e39e17e namespace: kserve-ci-e2e-test resourceVersion: "27527" uid: a44ab29c-f9d7-4869-97c4-9189260856fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afaec} kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: Started container splitter-graph-afaec metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c45fe50eaf6760 namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: 32d8f95b-3a8d-4c4b-86f8-ecc2bb459a71 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afaec} kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T18:08:03Z" message: Stopping container splitter-graph-afaec metadata: creationTimestamp: "2026-07-21T18:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:03Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c4605816ff0b20 namespace: kserve-ci-e2e-test resourceVersion: "30956" uid: bfbdf1ca-e716-41f2-b285-5ba4e7953acc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T18:08:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-afaec} kind: Pod name: splitter-graph-afaec-6f497d467-xk7jj namespace: kserve-ci-e2e-test resourceVersion: "27503" uid: 39988086-e8b6-43ed-bcac-947b2cf28d4e kind: Event lastTimestamp: "2026-07-21T18:08:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T18:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:31Z" name: splitter-graph-afaec-6f497d467-xk7jj.18c46058a2a4a989 namespace: kserve-ci-e2e-test resourceVersion: "31186" uid: 6338c95a-1d4b-47e4-a8ea-711b8802fad9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-afaec-6f497d467 namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: 50260914-de69-4f2a-991d-48366a9c0bc6 kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: 'Created pod: splitter-graph-afaec-6f497d467-xk7jj' metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec-6f497d467.18c45fe4ef88984c namespace: kserve-ci-e2e-test resourceVersion: "27504" uid: 5a596f60-0670-48b2-99a9-e3a5612c6238 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-afaec namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 5c9ce6c3-1229-4480-a087-c9215b499f25 kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: Scaled up replica set splitter-graph-afaec-6f497d467 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec.18c45fe4eeba673a namespace: kserve-ci-e2e-test resourceVersion: "27500" uid: 416d47fc-4d76-4dfa-bd29-a94e08415fa5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:49Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-afaec namespace: kserve-ci-e2e-test resourceVersion: "27517" uid: 2760f9c1-e05a-4151-b4c8-bab93b791469 kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: InferenceGraph [splitter-graph-afaec] is Ready metadata: creationTimestamp: "2026-07-21T17:59:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:59:49Z" name: splitter-graph-afaec.18c45fe4f375e986 namespace: kserve-ci-e2e-test resourceVersion: "27519" uid: 138b418c-000c-4afe-873a-b52c57621b63 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T18:00:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-afaec namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: 1971bb64-f59e-4a78-9064-143d1603259b kind: Event lastTimestamp: "2026-07-21T18:00:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T18:00:19Z" name: splitter-graph-afaec.18c45fe86d3c9db5 namespace: kserve-ci-e2e-test resourceVersion: "27745" uid: 324dd035-f6d1-49ca-ad0b-83c3583e3b0a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T18:00:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-afaec namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: 1971bb64-f59e-4a78-9064-143d1603259b kind: Event lastTimestamp: "2026-07-21T18:00:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T18:00:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T18:00:19Z" name: splitter-graph-afaec.18c45fe86d89c7f4 namespace: kserve-ci-e2e-test resourceVersion: "27746" uid: 60079e9b-f457-4b6e-a49e-0f904ea78f86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:58:38.895360Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26696" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-edfd2-6d56bf75d9-cj6n7 to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:58:38Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd48fc5d655 namespace: kserve-ci-e2e-test resourceVersion: "26701" uid: 8acfc25e-73da-4fed-85ca-5b825c167ad8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-edfd2-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:39Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd49b1b560d namespace: kserve-ci-e2e-test resourceVersion: "26714" uid: b0b59429-e370-4b79-a914-89106e20ac77 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:39Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:58:39Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd4cd3880b5 namespace: kserve-ci-e2e-test resourceVersion: "26723" uid: a9736bdf-b9a1-475c-b8f1-eff85f567d12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-edfd2} kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:39Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd4ce76c428 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: d84931b8-ba88-4e91-9626-2d889d8b7deb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-edfd2} kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:39Z" message: 'Created container: splitter-graph-edfd2' metadata: creationTimestamp: "2026-07-21T17:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:40Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd4d19714f9 namespace: kserve-ci-e2e-test resourceVersion: "26726" uid: f714f657-ad79-4f9f-a116-afdb0fb81a2d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-edfd2} kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:40Z" message: Started container splitter-graph-edfd2 metadata: creationTimestamp: "2026-07-21T17:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:40Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd4d2080694 namespace: kserve-ci-e2e-test resourceVersion: "26727" uid: 10b645a4-3e7c-4474-893a-30b8be8c6e6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-edfd2} kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:58:48Z" message: Stopping container splitter-graph-edfd2 metadata: creationTimestamp: "2026-07-21T17:58:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:48Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd6e88c000d namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 27efe897-1c58-4cc3-aec2-13518da89270 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-edfd2} kind: Pod name: splitter-graph-edfd2-6d56bf75d9-cj6n7 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74fa48df-8884-40e9-96e9-6c56969c7f52 kind: Event lastTimestamp: "2026-07-21T17:59:16Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:59:16Z" name: splitter-graph-edfd2-6d56bf75d9-cj6n7.18c45fd79299be7c namespace: kserve-ci-e2e-test resourceVersion: "27233" uid: 33aa11b9-c62e-46ce-83e4-2eaeeecbca46 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-edfd2-6d56bf75d9 namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: 246dba51-c4f5-4e6a-ae94-4f541b5f6bb8 kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: 'Created pod: splitter-graph-edfd2-6d56bf75d9-cj6n7' metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: splitter-graph-edfd2-6d56bf75d9.18c45fd48f584bbf namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 5347cb92-eb30-4e3b-bd41-61159742ef0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 883d0cb8-82e3-4d05-8344-c2cfd5edfd9e kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: Scaled up replica set splitter-graph-edfd2-6d56bf75d9 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: splitter-graph-edfd2.18c45fd48e61b547 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 3c38dae9-edba-4c8f-a392-06e1b3f445fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: f8d4d3c2-33b5-4d2f-a911-87d87e48b7f5 kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: InferenceGraph [splitter-graph-edfd2] is Ready metadata: creationTimestamp: "2026-07-21T17:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:38Z" name: splitter-graph-edfd2.18c45fd493d84b30 namespace: kserve-ci-e2e-test resourceVersion: "26712" uid: e03ba458-e753-46ca-8487-b3ecc616e3cb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:13.827766Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f4306d731ce namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 83e1591e-90f7-457d-b408-840d3fccb603 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f4321519299 namespace: kserve-ci-e2e-test resourceVersion: "21270" uid: 22cdc0ba-12e6-4189-bb22-1fa4a0aed8d5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f43229bec9e namespace: kserve-ci-e2e-test resourceVersion: "21272" uid: 5c15a8dc-7150-4a0b-bc45-cce0e337e747 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f43260d4acc namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: 9dfd2752-7225-4c29-b9c0-6ded75f83c1b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f4326c61e61 namespace: kserve-ci-e2e-test resourceVersion: "21274" uid: c47f365c-6b57-4524-8969-e2694724f165 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f4326d295e7 namespace: kserve-ci-e2e-test resourceVersion: "21275" uid: 73c03dd0-b69a-4f69-a9e2-b877b2c1de2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f432a525032 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 8cd4f926-9e76-433e-be86-8c542b1bd77b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:48:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:14Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f432acd0f75 namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: c8994145-2d25-4b50-bcc1-f36ca5f3046e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-21T17:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:57:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:41Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45f434e5ac8c0 namespace: kserve-ci-e2e-test resourceVersion: "26043" uid: fa3bf296-500d-414f-815f-4441f54b1eb1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45fc68fda4f46 namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: a5c2a31a-790d-44c6-9fc5-7b681ebcb60a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45fc68fdba8db namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: f4c8f32d-fd8b-427c-b114-e61b78b6acc7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg namespace: kserve-ci-e2e-test resourceVersion: "21223" uid: 584fd906-8440-4595-9371-2482e6946cdb kind: Event lastTimestamp: "2026-07-21T17:57:41Z" message: 'Readiness probe failed: Get "https://10.133.0.31:8643/healthz": dial tcp 10.133.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:41Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg.18c45fc716e2fb6d namespace: kserve-ci-e2e-test resourceVersion: "26041" uid: 519f3d26-accb-4597-86cc-18082ff33a0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4c9b0-predictor-698bd5c5d7 namespace: kserve-ci-e2e-test resourceVersion: "21212" uid: 533677ea-cb71-4c1f-8afc-6b13f8412495 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'Created pod: success-200-isvc-4c9b0-predictor-698bd5c5d7-z6hkg' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-4c9b0-predictor-698bd5c5d7.18c45f4306657162 namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 9a6a8f02-998d-4117-ab4b-084a3da48975 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 18d2872b-3ee2-4a76-b023-f882521dbb71 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Scaled up replica set success-200-isvc-4c9b0-predictor-698bd5c5d7 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-4c9b0-predictor.18c45f4304daacdb namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 806e627d-f95d-4b3f-9eb3-c41774ef2f8f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 4c8968aa-7b7b-4589-8182-6faef785e1ae kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: success-200-isvc-4c9b0-predictor.18c45f4683dc83e0 namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: 6e6979b3-e75a-4da5-a113-37b001d6f91c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 4c8968aa-7b7b-4589-8182-6faef785e1ae kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: success-200-isvc-4c9b0-predictor.18c45f4684735375 namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: 421bcf46-88e2-4ea8-8016-7e2d743d10db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 4c8968aa-7b7b-4589-8182-6faef785e1ae kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: success-200-isvc-4c9b0-predictor.18c45f4a032a81be namespace: kserve-ci-e2e-test resourceVersion: "22447" uid: f22eabc7-5171-471a-8bf4-2816f055c7f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4c9b0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 4c8968aa-7b7b-4589-8182-6faef785e1ae kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: success-200-isvc-4c9b0-predictor.18c45f4a038e2514 namespace: kserve-ci-e2e-test resourceVersion: "22448" uid: 97a27777-7b55-4296-a4a0-b74b315becdc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 53f2e11b-280d-4aca-85d8-6bd51437baa0 kind: Event lastTimestamp: "2026-07-21T17:49:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:11Z" name: success-200-isvc-4c9b0.18c45f43050d4256 namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 0b22d23c-5100-41e0-823a-a38a377e5482 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 53f2e11b-280d-4aca-85d8-6bd51437baa0 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-4c9b0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4c9b0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-4c9b0.18c45f4305844f3a namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 41e834e8-42e0-49cc-8a9e-3fdfa019a413 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21185" uid: 53f2e11b-280d-4aca-85d8-6bd51437baa0 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4c9b0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-4c9b0.18c45f4305847cca namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 7747ab6a-cd77-43c0-9296-110fcd1440a6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4c9b0 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 53f2e11b-280d-4aca-85d8-6bd51437baa0 kind: Event lastTimestamp: "2026-07-21T17:49:11Z" message: InferenceService [success-200-isvc-4c9b0] is Ready metadata: creationTimestamp: "2026-07-21T17:49:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:11Z" name: success-200-isvc-4c9b0.18c45f50607c8dab namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: b8e4c5b7-1a13-4c94-8584-8c233dfa2676 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:35:58.275375Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:35:58Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e97c480698f namespace: kserve-ci-e2e-test resourceVersion: "14712" uid: a59487d4-4d7c-46bb-808a-b47f4b555f87 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14715" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Add eth0 [10.133.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:35:58Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e97de5f943b namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 8ba5d686-ebc1-42d2-882f-4ef99208ee5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:35:58Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e97e0348a5d namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 0d43c3d4-80c8-421e-9b14-e15e6b101b2c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:22Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" in 24.155s (24.155s including waiting). Image size: 1335517280 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:22Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9d7ff65e28 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: f2b45728-303f-47c4-8378-3ec4e01ef525 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:36:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:22Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9d861233ce namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: 606e3e2d-d0f9-43c0-bdfc-2838c07a3c99 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9d8844e670 namespace: kserve-ci-e2e-test resourceVersion: "15082" uid: 4902a349-22f4-4e92-907f-4443ed185c93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-21T17:36:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:23Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9d88510af7 namespace: kserve-ci-e2e-test resourceVersion: "15083" uid: 49925baa-9c0e-472c-aad4-cb3645454d67 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 5.565s (5.565s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9ed40a9743 namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: edb85859-abb5-4bad-92ee-9ff6c156dbeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9ed8261b29 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 474274c4-696c-4fd3-853e-c002b3be57c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:36:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:36:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:36:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:36:28Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9ed8e934c7 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: c36d2533-6651-4748-9d2d-97e51484ad5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:36:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:37:15Z" message: 'Readiness probe failed: dial tcp 10.133.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:36:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:15Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45e9ef6243d98 namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: 16c7a7e1-9f20-47dc-b7d9-ea8fb6324ec5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:48Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45eb1732e3c01 namespace: kserve-ci-e2e-test resourceVersion: "15919" uid: 48a8ea01-262d-4f76-8874-f09b3a9db505 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:48Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45eb1732eb234 namespace: kserve-ci-e2e-test resourceVersion: "15920" uid: 2c462c7f-bab6-44cb-a83a-f9b5424696ad reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 2ecdb2de-9c9b-4548-bc44-c9cc92f1ec46 kind: Event lastTimestamp: "2026-07-21T17:37:50Z" message: 'Readiness probe failed: Get "https://10.133.0.18:8643/healthz": dial tcp 10.133.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:50Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7.18c45eb1d28dc9d8 namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 515a7134-dd1b-4c16-ab52-a6e62cb2833b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4dfe2-predictor-cf754f6b9 namespace: kserve-ci-e2e-test resourceVersion: "14693" uid: 6bbfb30b-3988-4c68-b22d-17cca5e78d12 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: 'Created pod: success-200-isvc-4dfe2-predictor-cf754f6b9-58tp7' metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: success-200-isvc-4dfe2-predictor-cf754f6b9.18c45e97c400a7df namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: f1bd8642-7f92-4db6-806e-b4e186a48fa8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: be1830e1-2979-4eec-b0ae-725046b45fe1 kind: Event lastTimestamp: "2026-07-21T17:35:58Z" message: Scaled up replica set success-200-isvc-4dfe2-predictor-cf754f6b9 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:35:58Z" name: success-200-isvc-4dfe2-predictor.18c45e97bbe4f6a2 namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 902033f6-2f7f-42c2-bd24-42a5f94fe09d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: c7cf4267-8209-4861-8e20-1619a93d309b kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: success-200-isvc-4dfe2-predictor.18c45e9b4177eccc namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 3793a3db-0230-4a3b-9feb-9a9c72c86ebe reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:36:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: c7cf4267-8209-4861-8e20-1619a93d309b kind: Event lastTimestamp: "2026-07-21T17:36:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:36:43Z" name: success-200-isvc-4dfe2-predictor.18c45e9b41e1683b namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 610bd7dc-f963-416f-be3f-10f44c86a055 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: c7cf4267-8209-4861-8e20-1619a93d309b kind: Event lastTimestamp: "2026-07-21T17:37:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:43Z" name: success-200-isvc-4dfe2-predictor.18c45ea5bedab7e0 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: f2bb6e93-c541-4dfb-bc97-5c9659b10027 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:36:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dfe2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: c7cf4267-8209-4861-8e20-1619a93d309b kind: Event lastTimestamp: "2026-07-21T17:37:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:36:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:43Z" name: success-200-isvc-4dfe2-predictor.18c45ea5bf4d68b3 namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: 0c614ca6-13ae-404d-b406-414c7aeb1c92 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:35:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 7095256d-7aa8-415e-9ea2-c41971b1af0e kind: Event lastTimestamp: "2026-07-21T17:37:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:35:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:25Z" name: success-200-isvc-4dfe2.18c45e97c21ceee1 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: ef64a922-ece4-4f80-9a63-8c48d61d0d02 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 7095256d-7aa8-415e-9ea2-c41971b1af0e kind: Event lastTimestamp: "2026-07-21T17:37:25Z" message: InferenceService [success-200-isvc-4dfe2] is Ready metadata: creationTimestamp: "2026-07-21T17:37:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:25Z" name: success-200-isvc-4dfe2.18c45eac05c2654c namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 2d25f058-3b0f-4309-8804-650836a276fe reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:37:48.682928Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:37:48Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb1794e394b namespace: kserve-ci-e2e-test resourceVersion: "15950" uid: 68c5a2ac-9947-4d74-9137-28cd214d1be0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb1932620d7 namespace: kserve-ci-e2e-test resourceVersion: "15998" uid: 64d7e578-3d0f-47c3-b5ab-9f672ea25d8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb194e0a636 namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 5be108ea-a021-4f48-a5b1-295f35d71828 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb19844d1ca namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 16b3df53-95d0-463a-9bd7-f89dff5f7e27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb198a8c4b1 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: cfd718f6-91d3-4eda-b3a7-caa4b5c0bd34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb198b20ba2 namespace: kserve-ci-e2e-test resourceVersion: "16003" uid: 2a6ff8f0-9124-4c39-8103-6b8031cc6d65 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb19b623039 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: acef71fd-c9a7-40fa-a46f-edd923e014c2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:37:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:49Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb19bd1627f namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: faada53d-102d-4566-8ba8-bfe89219dc80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:37:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:47:05Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:05Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45eb1aadc9834 namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: c5b00ec5-3543-4ae5-b216-90e152053422 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45f32a12b586f namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 443c8124-6dfb-46d2-b58b-97ef42811f93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45f32a12bf121 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 589b4727-e9d1-4643-b7b0-d785832d8d03 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6 namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: aeeeed18-d5cb-44dc-98d8-e896038dee1b kind: Event lastTimestamp: "2026-07-21T17:47:05Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:05Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6.18c45f331f7719bd namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: 1b450ca1-17f0-483e-97fc-26d24b0a654e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5bf12-predictor-5dd8fcc8cf namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: b79bc3a5-267a-4f87-be58-e236961bec02 kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: 'Created pod: success-200-isvc-5bf12-predictor-5dd8fcc8cf-qn9n6' metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: success-200-isvc-5bf12-predictor-5dd8fcc8cf.18c45eb178de7629 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 92cc69b7-8748-4ca7-b020-edb320bd3467 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 7168c207-c073-4f5b-9089-fb4be5053b6f kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Scaled up replica set success-200-isvc-5bf12-predictor-5dd8fcc8cf from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:48Z" name: success-200-isvc-5bf12-predictor.18c45eb175dbd719 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: d89eb851-f896-444b-9bed-c76cd035f704 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 8774cb12-c6ea-4f08-a4b0-6563bde803a8 kind: Event lastTimestamp: "2026-07-21T17:38:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:33Z" name: success-200-isvc-5bf12-predictor.18c45eb4f46ea0a3 namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: 33848452-055e-4fb5-b3a8-f352a7bba81f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:38:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: 8774cb12-c6ea-4f08-a4b0-6563bde803a8 kind: Event lastTimestamp: "2026-07-21T17:38:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:33Z" name: success-200-isvc-5bf12-predictor.18c45eb4f4e81111 namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: 76e69bfe-f4a0-473b-9539-527ee391ef46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 8774cb12-c6ea-4f08-a4b0-6563bde803a8 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: success-200-isvc-5bf12-predictor.18c45ebf715f95e8 namespace: kserve-ci-e2e-test resourceVersion: "16915" uid: f262d3ea-99e3-42f2-8f39-bc3554def454 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5bf12-predictor namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: 8774cb12-c6ea-4f08-a4b0-6563bde803a8 kind: Event lastTimestamp: "2026-07-21T17:39:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:03Z" name: success-200-isvc-5bf12-predictor.18c45ebf71c58de5 namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: 1679c7fe-c9d8-4dec-9e63-524620326e05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: 2ba8819e-ec84-474a-aef4-02e1992a53d2 kind: Event lastTimestamp: "2026-07-21T17:38:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:35Z" name: success-200-isvc-5bf12.18c45eb175dfa751 namespace: kserve-ci-e2e-test resourceVersion: "16645" uid: f98c0df5-a732-46f2-9f7d-a39b10c1d59c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 2ba8819e-ec84-474a-aef4-02e1992a53d2 kind: Event lastTimestamp: "2026-07-21T17:38:35Z" message: InferenceService [success-200-isvc-5bf12] is Ready metadata: creationTimestamp: "2026-07-21T17:38:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:35Z" name: success-200-isvc-5bf12.18c45ebc66f3a12f namespace: kserve-ci-e2e-test resourceVersion: "16644" uid: c9a661a5-1037-4ff9-b4d0-6d93a4bce78a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:38:18.550321Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16397" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:38:18Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb86d8a3e00 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 0a8781f4-8bd2-4ed8-86e0-cb96ddeea89f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8558d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb87543f264 namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: de8d9a62-7376-4342-86a5-62fd6e7f08f5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8ab9d7f2e namespace: kserve-ci-e2e-test resourceVersion: "16467" uid: 9f90aa8d-ebbd-4ba0-8e85-8ba1f512af41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8ad21c020 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: c55ac187-2fc0-4759-8260-6aa834b78421 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8b10bbbc1 namespace: kserve-ci-e2e-test resourceVersion: "16472" uid: 84af8455-02da-4049-90c7-a44a4a29c002 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8b18c11b3 namespace: kserve-ci-e2e-test resourceVersion: "16473" uid: d14be37f-86d3-45bf-a2ed-73e9d102bfc4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8b197a64a namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: adb6949f-c4af-4163-a8b1-5164554aad64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8b6036b0a namespace: kserve-ci-e2e-test resourceVersion: "16478" uid: e1248769-a42d-415c-9997-68beb9d2171b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:38:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:38:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:19Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8b67d6c8b namespace: kserve-ci-e2e-test resourceVersion: "16479" uid: 1afd7ee4-632c-4b2b-8d2f-0efc391fe2ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:38:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:47:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:38:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:36Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45eb8ec2c4221 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 1f944350-608e-4db8-b081-1c8cc5e3c05c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45f39a1ae9411 namespace: kserve-ci-e2e-test resourceVersion: "20649" uid: 7965f23e-3186-4498-9b22-12c4a160dc35 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45f39a1b4b442 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 43a0db40-6d81-4400-a196-5470d5a8b607 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: 9de3561f-aecd-4d01-afa8-60277198d9d1 kind: Event lastTimestamp: "2026-07-21T17:47:36Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:36Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm.18c45f3a609b51c4 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 2f2e22ce-8c12-4e4a-bfef-dcb8e5e731ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8558d-predictor-6df9dd9cf5 namespace: kserve-ci-e2e-test resourceVersion: "16381" uid: 8432f52d-2880-4b3b-84ea-ad7066b5c821 kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'Created pod: success-200-isvc-8558d-predictor-6df9dd9cf5-bn5nm' metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: success-200-isvc-8558d-predictor-6df9dd9cf5.18c45eb86d1a72f3 namespace: kserve-ci-e2e-test resourceVersion: "16404" uid: 6ed0ba6c-64d3-4fbf-ab9c-7e483f2ff9a8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: abdc65d1-8278-4d8a-b240-6ea2f7336aca kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: Scaled up replica set success-200-isvc-8558d-predictor-6df9dd9cf5 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:18Z" name: success-200-isvc-8558d-predictor.18c45eb86b63713d namespace: kserve-ci-e2e-test resourceVersion: "16383" uid: dcc96685-7427-444a-b49e-ee7ccb4221b1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: f309abce-4744-4465-9b9d-d4975c0f4513 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: success-200-isvc-8558d-predictor.18c45ebbea240887 namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: e64a8289-516a-4f48-8087-549e5e3c9fb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:38:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16380" uid: f309abce-4744-4465-9b9d-d4975c0f4513 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:38:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: success-200-isvc-8558d-predictor.18c45ebbea96a355 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: 1f0b5b49-1641-40c3-8fed-cfba3be81334 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: f309abce-4744-4465-9b9d-d4975c0f4513 kind: Event lastTimestamp: "2026-07-21T17:39:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:33Z" name: success-200-isvc-8558d-predictor.18c45ec2e8cba15b namespace: kserve-ci-e2e-test resourceVersion: "17196" uid: 24cf9791-6e5e-412c-b8e0-e92a48cb8622 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8558d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: f309abce-4744-4465-9b9d-d4975c0f4513 kind: Event lastTimestamp: "2026-07-21T17:39:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:33Z" name: success-200-isvc-8558d-predictor.18c45ec2e9399caf namespace: kserve-ci-e2e-test resourceVersion: "17197" uid: dac23a9c-2d38-44d4-9d33-dec838abaa20 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16355" uid: 59547dc4-2c91-4a5a-9876-99f8edc143fc kind: Event lastTimestamp: "2026-07-21T17:39:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:39:06Z" name: success-200-isvc-8558d.18c45eb86b72f4ab namespace: kserve-ci-e2e-test resourceVersion: "16948" uid: 7348556f-dc5b-40a1-bf0f-86b1ff3ad098 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:39:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8558d namespace: kserve-ci-e2e-test resourceVersion: "16946" uid: 59547dc4-2c91-4a5a-9876-99f8edc143fc kind: Event lastTimestamp: "2026-07-21T17:39:06Z" message: InferenceService [success-200-isvc-8558d] is Ready metadata: creationTimestamp: "2026-07-21T17:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:39:06Z" name: success-200-isvc-8558d.18c45ec3a7b00ee5 namespace: kserve-ci-e2e-test resourceVersion: "16947" uid: 4730a2f2-4a89-4ad5-a6ae-1f816da1583c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:48:43.848864Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8e991-predictor-799cbc5d5f-vzztz to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a043cd097 namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: ca3ee77d-2718-43cb-a485-049f948a1a7a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a1e53c8c3 namespace: kserve-ci-e2e-test resourceVersion: "21719" uid: 0a09cdd6-a664-4343-89bb-f432c07fcc77 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a1fc95365 namespace: kserve-ci-e2e-test resourceVersion: "21721" uid: 564a4ad6-6124-4798-a5d9-fa86da68c95e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a23283971 namespace: kserve-ci-e2e-test resourceVersion: "21722" uid: 391f83d1-3a2d-47aa-a824-d6c6146d8898 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a23b29a24 namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 44a42ddd-4958-4215-99ae-92c07731ed0a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a23be684b namespace: kserve-ci-e2e-test resourceVersion: "21725" uid: 8456e85f-a17f-436b-92f3-25522ec513a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a269fa330 namespace: kserve-ci-e2e-test resourceVersion: "21729" uid: 12ff6625-c7eb-4744-b3b5-467ac15b9103 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:48:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:44Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a27203629 namespace: kserve-ci-e2e-test resourceVersion: "21730" uid: 5b5754f5-9a67-4cb4-af4f-78bbe9d65afd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:58:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:01Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45f4a5288410d namespace: kserve-ci-e2e-test resourceVersion: "26346" uid: 62aca23d-e7f6-45af-9e78-80a5adf7a2ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45fcb2de2f32d namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: 4526ac16-40fe-4718-82e1-3c376bcfb06e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45fcb2de386fe namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 05dd94a6-dbfd-46a2-998d-55c6a0b80993 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz namespace: kserve-ci-e2e-test resourceVersion: "21665" uid: 9622b1bb-2668-4e94-a02f-e1c605d3a187 kind: Event lastTimestamp: "2026-07-21T17:58:01Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:01Z" name: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz.18c45fcbc7053b47 namespace: kserve-ci-e2e-test resourceVersion: "26345" uid: effdbb33-fd6d-4c89-bdc9-285bdcd072fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8e991-predictor-799cbc5d5f namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: ac508e5d-5808-422f-8c34-b845bd584349 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'Created pod: success-200-isvc-8e991-predictor-799cbc5d5f-vzztz' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-8e991-predictor-799cbc5d5f.18c45f4a03bf1a50 namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: f7cb206e-6e35-4602-b2b3-acf9b8ada319 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 6754b865-8f39-4460-8fb7-4681e4a8e73b kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Scaled up replica set success-200-isvc-8e991-predictor-799cbc5d5f from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-8e991-predictor.18c45f4a01b1b8ff namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: dc6e75f2-c463-4d78-807b-1e484bcfbd5e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 87b97602-a351-435b-9b74-78a6f01678ca kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: success-200-isvc-8e991-predictor.18c45f4d8047be32 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: a0a5d42a-22d8-4e48-91a5-fc90ae0c766b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 87b97602-a351-435b-9b74-78a6f01678ca kind: Event lastTimestamp: "2026-07-21T17:48:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:48:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:58Z" name: success-200-isvc-8e991-predictor.18c45f4d80b8c5eb namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 71974c1d-6989-4017-adf3-f050f1b8aa1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 87b97602-a351-435b-9b74-78a6f01678ca kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: success-200-isvc-8e991-predictor.18c45f5100121c40 namespace: kserve-ci-e2e-test resourceVersion: "22441" uid: 42560244-c5dc-432b-a65b-8529a47970c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-21T17:49:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8e991-predictor namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 87b97602-a351-435b-9b74-78a6f01678ca kind: Event lastTimestamp: "2026-07-21T17:49:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:49:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:49:58Z" name: success-200-isvc-8e991-predictor.18c45f51007af93b namespace: kserve-ci-e2e-test resourceVersion: "22442" uid: 399e276c-8e15-4ed5-be93-d76d179580a9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: f5cbeaa6-071f-4fc3-80f9-8ab188d49f67 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:31Z" name: success-200-isvc-8e991.18c45f4a01f30f1c namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 4a299231-5131-4235-9404-62a964b84b6a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: f5cbeaa6-071f-4fc3-80f9-8ab188d49f67 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-8e991": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8e991": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-8e991.18c45f4a026afa0d namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: defe0aa4-d646-49fd-88a6-0c93fe6ec93b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: f5cbeaa6-071f-4fc3-80f9-8ab188d49f67 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8e991": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-8e991.18c45f4a026b67a8 namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 615d16c3-4e6b-4d38-85c1-73b5ddbc699e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:49:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8e991 namespace: kserve-ci-e2e-test resourceVersion: "22183" uid: f5cbeaa6-071f-4fc3-80f9-8ab188d49f67 kind: Event lastTimestamp: "2026-07-21T17:49:31Z" message: InferenceService [success-200-isvc-8e991] is Ready metadata: creationTimestamp: "2026-07-21T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:49:31Z" name: success-200-isvc-8e991.18c45f550d974455 namespace: kserve-ci-e2e-test resourceVersion: "22184" uid: 76f7690b-26e8-41ce-b893-6459fb1f55c7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:57:58.690833Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb3365664e namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 0cdd4bf4-c359-48b4-bdc4-e70fd0fb7984 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-984cf-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb424dd30f namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 44a81889-e101-42aa-9b09-6ecc8110dbcc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb7117ad25 namespace: kserve-ci-e2e-test resourceVersion: "26320" uid: 6b0822f9-07bd-4b34-bf88-24c0cd1dda15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb728e71d1 namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: a3e4211a-cfd2-41d4-a922-25d835eb929d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb758d1b9c namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: 9c90d84b-c7e0-460b-a202-6a55ed65afeb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb7605c142 namespace: kserve-ci-e2e-test resourceVersion: "26324" uid: 6067793a-5575-4be3-b965-c085318e4dcf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb760fac0c namespace: kserve-ci-e2e-test resourceVersion: "26325" uid: 063f3366-9c3f-4924-ae41-5010d95acf99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb78fef743 namespace: kserve-ci-e2e-test resourceVersion: "26326" uid: 122ba704-70f2-44d1-8a0a-531401932ea1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:57:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:59Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcb7976269b namespace: kserve-ci-e2e-test resourceVersion: "26327" uid: 86c8b6d3-c663-4788-8de6-cafcc17990ad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T17:58:37Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:37Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c45fcbce2b803d namespace: kserve-ci-e2e-test resourceVersion: "26672" uid: 26f01d4f-d318-45bc-b2b0-615630a274fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T18:15:18Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T18:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:18Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c460bd3eaae785 namespace: kserve-ci-e2e-test resourceVersion: "33982" uid: a42dcdf3-c1bc-407a-94f6-0a94b635d065 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 59d6e22f-3560-457e-8d5e-95cd24157bb9 kind: Event lastTimestamp: "2026-07-21T18:15:18Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T18:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:18Z" name: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9.18c460bd3eab75a0 namespace: kserve-ci-e2e-test resourceVersion: "33983" uid: e3e84ef3-53a3-4c92-9f60-faa537d6d16b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-984cf-predictor-7bdc6476c9 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 925d2696-6e35-4851-8d10-e4a6b1e23d9e kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'Created pod: success-200-isvc-984cf-predictor-7bdc6476c9-nrlz9' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf-predictor-7bdc6476c9.18c45fcb32f7efb0 namespace: kserve-ci-e2e-test resourceVersion: "26256" uid: 617b2f62-cd70-45b4-901c-0f34993788a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: abab6825-bd7b-4e91-b11d-979a4fceef2d kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: Scaled up replica set success-200-isvc-984cf-predictor-7bdc6476c9 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf-predictor.18c45fcb317676c9 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: 8b5a9bff-5a04-40cc-bc55-222c2177d4aa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 4b38ec9a-e22c-48dd-842e-26600b98c6a2 kind: Event lastTimestamp: "2026-07-21T17:58:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:43Z" name: success-200-isvc-984cf-predictor.18c45fceaff5ee0f namespace: kserve-ci-e2e-test resourceVersion: "26757" uid: c4684e64-2362-4889-8b99-4a5ac769ea03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 4b38ec9a-e22c-48dd-842e-26600b98c6a2 kind: Event lastTimestamp: "2026-07-21T17:58:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:43Z" name: success-200-isvc-984cf-predictor.18c45fceb037b038 namespace: kserve-ci-e2e-test resourceVersion: "26758" uid: c42e2e17-a423-469e-a916-0008dd97c611 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 4b38ec9a-e22c-48dd-842e-26600b98c6a2 kind: Event lastTimestamp: "2026-07-21T17:59:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:13Z" name: success-200-isvc-984cf-predictor.18c45fd92c836ae6 namespace: kserve-ci-e2e-test resourceVersion: "27200" uid: 2d47b922-4186-46fb-86f0-8275357f916f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-984cf-predictor namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 4b38ec9a-e22c-48dd-842e-26600b98c6a2 kind: Event lastTimestamp: "2026-07-21T17:59:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:13Z" name: success-200-isvc-984cf-predictor.18c45fd92ce7b2d1 namespace: kserve-ci-e2e-test resourceVersion: "27201" uid: b5227c4e-2188-418d-a4c9-22d187bfd272 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: efd6d0fb-2218-4037-8d0c-63f4cf131dcf kind: Event lastTimestamp: "2026-07-21T17:58:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:47Z" name: success-200-isvc-984cf.18c45fcb31a381d8 namespace: kserve-ci-e2e-test resourceVersion: "26804" uid: d4cf79cc-ac8c-4418-b8c7-bea10bc91fd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: efd6d0fb-2218-4037-8d0c-63f4cf131dcf kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-984cf": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-984cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf.18c45fcb3214a1d4 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 7552f16b-dfb3-4fe9-903e-2a63a2bad2d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: efd6d0fb-2218-4037-8d0c-63f4cf131dcf kind: Event lastTimestamp: "2026-07-21T17:57:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-984cf": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:58Z" name: success-200-isvc-984cf.18c45fcb3214d6f0 namespace: kserve-ci-e2e-test resourceVersion: "26254" uid: 6be73ec0-1bab-4851-b539-015f77402429 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-984cf namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: efd6d0fb-2218-4037-8d0c-63f4cf131dcf kind: Event lastTimestamp: "2026-07-21T17:58:47Z" message: InferenceService [success-200-isvc-984cf] is Ready metadata: creationTimestamp: "2026-07-21T17:58:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:47Z" name: success-200-isvc-984cf.18c45fd689a1fdc4 namespace: kserve-ci-e2e-test resourceVersion: "26803" uid: 046beebf-9743-4c5d-8a49-3519088faae3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:58:49.153890Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd6f33a83b4 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 3fc2b31b-5ecd-41e1-a99b-2137ca6f5542 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-afaec-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd6fde8a990 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 6432241a-2e30-4142-b734-8d0f8a4d4055 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd730a40b28 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 201cab9e-c859-47c6-bb21-6a77e205c4bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd732185951 namespace: kserve-ci-e2e-test resourceVersion: "26945" uid: fdca9a8c-e0cf-4182-a655-b88da648bd95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd735c7caea namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: 38e02ec4-fa41-4045-8c67-562641e9c303 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd7365634fe namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: 32fcb369-9799-40fc-93e0-b09c700a8ea8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd736628c28 namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: e7c9de00-6b6c-4588-b0b9-7946821b3742 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd73a442783 namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 375ef0f6-3100-4e5b-b09c-ecaa1b148910 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:58:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:50Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd73ac1e46a namespace: kserve-ci-e2e-test resourceVersion: "26956" uid: 26ca9004-f3a0-48da-a1cf-aa3d8b63c579 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T17:59:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:59:27Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c45fd77ef370cd namespace: kserve-ci-e2e-test resourceVersion: "27315" uid: e726e2e0-428b-46b3-a2e2-c2bf4c3af490 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T18:08:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T18:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:03Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c460581f15dbd9 namespace: kserve-ci-e2e-test resourceVersion: "30972" uid: b5f48c07-f318-44c5-9fce-f447d85d5ca0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:08:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: 1827ba9b-0f13-4f48-bf1e-66cc74ba519a kind: Event lastTimestamp: "2026-07-21T18:08:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T18:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:08:03Z" name: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx.18c460581f165f5c namespace: kserve-ci-e2e-test resourceVersion: "30973" uid: 479bd85e-295c-4ffe-8b44-0b93bb018724 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-afaec-predictor-66b7f9bdff namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 68fcf164-4e06-4157-b8d6-d882ad31576e kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'Created pod: success-200-isvc-afaec-predictor-66b7f9bdff-rm6wx' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec-predictor-66b7f9bdff.18c45fd6f2e06f91 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 2d1ac22a-19aa-45b4-bab6-2e0656771f89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: b06919e3-c2a5-4f0d-9710-3c864e11c8cf kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Scaled up replica set success-200-isvc-afaec-predictor-66b7f9bdff from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec-predictor.18c45fd6f13a7ec6 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: a2e5c39e-fde6-4595-be4f-5b8d578ba47c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: 32a60bde-1221-43b7-9c94-93387ff37440 kind: Event lastTimestamp: "2026-07-21T17:59:19Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:19Z" name: success-200-isvc-afaec-predictor.18c45fda6fb8428b namespace: kserve-ci-e2e-test resourceVersion: "27252" uid: ac07530d-7fe0-4933-a37a-500220a3a592 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:04Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: 32a60bde-1221-43b7-9c94-93387ff37440 kind: Event lastTimestamp: "2026-07-21T17:59:19Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:19Z" name: success-200-isvc-afaec-predictor.18c45fda70074822 namespace: kserve-ci-e2e-test resourceVersion: "27253" uid: 9f94180c-d983-4f9e-9c01-d9060a73b0a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: 32a60bde-1221-43b7-9c94-93387ff37440 kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:49Z" name: success-200-isvc-afaec-predictor.18c45fe16d8f192a namespace: kserve-ci-e2e-test resourceVersion: "27491" uid: 3ed30a16-c4d0-476f-819d-64f61f5cd5e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:59:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-afaec-predictor namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: 32a60bde-1221-43b7-9c94-93387ff37440 kind: Event lastTimestamp: "2026-07-21T17:59:49Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:59:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:49Z" name: success-200-isvc-afaec-predictor.18c45fe16e024853 namespace: kserve-ci-e2e-test resourceVersion: "27493" uid: 58add13c-bcdb-49a3-8ff5-c3a4bf6f93a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 2dcd93d5-128c-442a-8d01-53ed96e6e3ea kind: Event lastTimestamp: "2026-07-21T17:59:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:59:37Z" name: success-200-isvc-afaec.18c45fd6f13b19ac namespace: kserve-ci-e2e-test resourceVersion: "27406" uid: c5cc6da9-4e7a-4cd5-92b9-7bf804db00be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 2dcd93d5-128c-442a-8d01-53ed96e6e3ea kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'Failed to update status for InferenceService "success-200-isvc-afaec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-afaec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec.18c45fd6f2088d30 namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: 0a722527-aea8-4cd4-9495-c61e567bddad reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 2dcd93d5-128c-442a-8d01-53ed96e6e3ea kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-afaec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-afaec.18c45fd6f208f03f namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 570a4da7-afc8-4b8f-b396-3d9f949f2b7b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:59:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-afaec namespace: kserve-ci-e2e-test resourceVersion: "27404" uid: 2dcd93d5-128c-442a-8d01-53ed96e6e3ea kind: Event lastTimestamp: "2026-07-21T17:59:37Z" message: InferenceService [success-200-isvc-afaec] is Ready metadata: creationTimestamp: "2026-07-21T17:59:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:59:37Z" name: success-200-isvc-afaec.18c45fe23a90a367 namespace: kserve-ci-e2e-test resourceVersion: "27405" uid: c3ffa6fa-725d-435c-b7fb-abda31cc673f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:33.540281Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20683" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b64c9-predictor-765f6b789f-47rtc to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39a584f101 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 2b6dcc79-e16e-4db6-991f-60def51a1633 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b64c9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39b13e0fa7 namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: 60bc2a5e-27cb-4b3c-8cda-763113c298b9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20694" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39e312a4dc namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 122ecf76-089a-495c-a2cb-1b0a40312c15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39e467508c namespace: kserve-ci-e2e-test resourceVersion: "20751" uid: 7892727a-5884-4ba0-b78d-970dfca3698a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39e86b676b namespace: kserve-ci-e2e-test resourceVersion: "20752" uid: 42516b9c-b1c8-4703-a101-bc1f71b2f319 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39e8e910ac namespace: kserve-ci-e2e-test resourceVersion: "20753" uid: 81e8cc15-861a-42f3-8f10-5d56f31451e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39e8f55caf namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: d0f2ca8f-92ad-48a8-aa58-d9786a419616 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39ec3cdf54 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: e9cce772-e596-4dd0-878b-380e162e4ada reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:47:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:34Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39ecb0d37a namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: d2e0c58e-ae08-4945-a4c1-9318864fd5f0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-21T17:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:48:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:11Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f39f401dfc9 namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: a2dac193-408f-4031-8869-606994f64b5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f49ffdf0ac5 namespace: kserve-ci-e2e-test resourceVersion: "21628" uid: 0b3d95a1-c05e-4bec-ba3f-07a005a9313a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:48:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:43Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f49ffdfa74a namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 6b5bcce1-718b-4bc6-86ac-d8be0e72687c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 07fc8a39-1f80-4327-bf72-32bb289206e5 kind: Event lastTimestamp: "2026-07-21T17:48:46Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:46Z" name: success-200-isvc-b64c9-predictor-765f6b789f-47rtc.18c45f4ab81c116d namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 3eeaf1e3-65f1-483e-9c7f-cf637c1cc123 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b64c9-predictor-765f6b789f namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 13d80167-826d-417e-9a58-33b1230c19d1 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'Created pod: success-200-isvc-b64c9-predictor-765f6b789f-47rtc' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9-predictor-765f6b789f.18c45f39a52c38a1 namespace: kserve-ci-e2e-test resourceVersion: "20686" uid: ea9e088c-ef29-4cdf-904e-6d8579a7dbdc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: d27cb38f-ba15-4707-9908-799160d0e3f8 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: Scaled up replica set success-200-isvc-b64c9-predictor-765f6b789f from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9-predictor.18c45f39a36f9d70 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: c9b2cfdc-1e5a-4735-8a57-77e0f0166ae0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 9c4bb1e0-c6d3-402c-bd88-c82749374ee3 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: success-200-isvc-b64c9-predictor.18c45f3d227586d6 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: dcd5dcba-657b-4346-b738-fa6004b9c1cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 9c4bb1e0-c6d3-402c-bd88-c82749374ee3 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: success-200-isvc-b64c9-predictor.18c45f3d22e22997 namespace: kserve-ci-e2e-test resourceVersion: "21037" uid: e083ce3d-43de-4e3c-8387-231dedcb501f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 9c4bb1e0-c6d3-402c-bd88-c82749374ee3 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: success-200-isvc-b64c9-predictor.18c45f4420779725 namespace: kserve-ci-e2e-test resourceVersion: "21493" uid: 4d95729e-b6b1-443e-b96a-35fd217c49f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:48:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b64c9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 9c4bb1e0-c6d3-402c-bd88-c82749374ee3 kind: Event lastTimestamp: "2026-07-21T17:48:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:33Z" name: success-200-isvc-b64c9-predictor.18c45f4420e6e826 namespace: kserve-ci-e2e-test resourceVersion: "21495" uid: 58ac8f9a-f9e0-46e8-8f81-35b45408cd40 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a1b7733d-1df2-4750-a51e-a64eb3e901e3 kind: Event lastTimestamp: "2026-07-21T17:48:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:22Z" name: success-200-isvc-b64c9.18c45f39a36588b2 namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: fc610f51-f0f7-49b7-8bc6-d9fdfb45eaef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a1b7733d-1df2-4750-a51e-a64eb3e901e3 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'Failed to update status for InferenceService "success-200-isvc-b64c9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b64c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9.18c45f39a41ca428 namespace: kserve-ci-e2e-test resourceVersion: "20677" uid: e09064c6-bf18-430b-a3d2-eb205d0ccd4c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a1b7733d-1df2-4750-a51e-a64eb3e901e3 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b64c9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:47:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-b64c9.18c45f39a41d096e namespace: kserve-ci-e2e-test resourceVersion: "20680" uid: 9e6ec6cc-1824-4314-a5f7-67e89469cfbd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b64c9 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: a1b7733d-1df2-4750-a51e-a64eb3e901e3 kind: Event lastTimestamp: "2026-07-21T17:48:21Z" message: InferenceService [success-200-isvc-b64c9] is Ready metadata: creationTimestamp: "2026-07-21T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:48:21Z" name: success-200-isvc-b64c9.18c45f44eb0c72b4 namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: bdaf4fa3-4810-4818-9da2-f44742fb8bbd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:57:38.838544Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25942" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6941bc38c namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: 7ea2adf5-c5e6-4a4d-9fed-827cfb823233 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-edfd2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc69c05e7bd namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: 577f1c8b-d6ad-4d1f-a00f-eef3a3dd9acb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25969" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:57:39Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6d1e3679e namespace: kserve-ci-e2e-test resourceVersion: "26011" uid: 804b5ec2-bbc8-4ade-a73b-de946e9129f7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6d33eae09 namespace: kserve-ci-e2e-test resourceVersion: "26013" uid: 46c9f6f2-17fd-4262-b125-93c27210b92d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6d76d54cd namespace: kserve-ci-e2e-test resourceVersion: "26017" uid: 91a1ec86-302c-43dd-8c90-3040afaaca82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6d7f92386 namespace: kserve-ci-e2e-test resourceVersion: "26019" uid: cfa0dc05-2fec-4aa9-87ed-e3f046444fe6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:57:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:39Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6d82a252e namespace: kserve-ci-e2e-test resourceVersion: "26020" uid: 37904b8d-13f7-4e30-a8c5-3630213b44de reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6daa07511 namespace: kserve-ci-e2e-test resourceVersion: "26022" uid: 474c8881-8091-43ee-829a-2a6473d6f168 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:57:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:57:40Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6db1eb485 namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: 6a3a00a5-6fd6-4b87-b9df-f2a06b3bb98a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:58:16Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:16Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fc6e48dae95 namespace: kserve-ci-e2e-test resourceVersion: "26494" uid: 7216f5b9-a47a-40a7-b458-fa7c497dc210 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fd6ef44d1fa namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: be2cfac9-c49d-4476-9c69-ebe9f5c77c1b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:58:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:49Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fd6ef4545ff namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 1694e0a1-3615-474e-938e-1b6714378ef6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p namespace: kserve-ci-e2e-test resourceVersion: "25947" uid: e7f959bd-b166-4bf3-9d39-061839b2f5fa kind: Event lastTimestamp: "2026-07-21T17:58:51Z" message: 'Readiness probe failed: Get "https://10.133.0.35:8643/healthz": dial tcp 10.133.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:51Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p.18c45fd76cb06783 namespace: kserve-ci-e2e-test resourceVersion: "26973" uid: 496debb8-0e14-497b-a7e6-34836fb7d1f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-edfd2-predictor-6f4b9cd55c namespace: kserve-ci-e2e-test resourceVersion: "25930" uid: 4ad7cdf4-4844-4a55-b3af-acea9f7a4c4b kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'Created pod: success-200-isvc-edfd2-predictor-6f4b9cd55c-n6g4p' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2-predictor-6f4b9cd55c.18c45fc6937a5ead namespace: kserve-ci-e2e-test resourceVersion: "25946" uid: b91cd807-5777-4fca-bd69-0dfddd60dc01 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25926" uid: a9410aae-ccd2-44e9-8fc7-155611a7f389 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: Scaled up replica set success-200-isvc-edfd2-predictor-6f4b9cd55c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2-predictor.18c45fc68fd0cc2b namespace: kserve-ci-e2e-test resourceVersion: "25932" uid: 19b713f1-33a3-44d4-ad3a-9f30ba8b4bbd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: dea8341f-2976-449b-a12f-0f7310a06e8d kind: Event lastTimestamp: "2026-07-21T17:57:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:53Z" name: success-200-isvc-edfd2-predictor.18c45fca0e672d26 namespace: kserve-ci-e2e-test resourceVersion: "26148" uid: 5e891fee-0b10-4c6a-a5a3-d2165b685f74 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: dea8341f-2976-449b-a12f-0f7310a06e8d kind: Event lastTimestamp: "2026-07-21T17:57:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:57:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:57:53Z" name: success-200-isvc-edfd2-predictor.18c45fca0ecdba4d namespace: kserve-ci-e2e-test resourceVersion: "26150" uid: 609f5b73-b3c4-4504-bc33-feaa5d7f2733 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: dea8341f-2976-449b-a12f-0f7310a06e8d kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: success-200-isvc-edfd2-predictor.18c45fcd8e535598 namespace: kserve-ci-e2e-test resourceVersion: "26685" uid: d1b83b37-127d-4a76-836a-0fe73dfc4ed0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-edfd2-predictor namespace: kserve-ci-e2e-test resourceVersion: "26149" uid: dea8341f-2976-449b-a12f-0f7310a06e8d kind: Event lastTimestamp: "2026-07-21T17:58:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:38Z" name: success-200-isvc-edfd2-predictor.18c45fcd8ec43f2f namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 65399561-6ff7-4b8a-8c25-40de21003415 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 49385531-e6f8-4d7e-85ad-dea228eec602 kind: Event lastTimestamp: "2026-07-21T17:58:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:26Z" name: success-200-isvc-edfd2.18c45fc68fd29986 namespace: kserve-ci-e2e-test resourceVersion: "26579" uid: 289c8d79-0e29-4c94-bc59-6de3af8e9946 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 49385531-e6f8-4d7e-85ad-dea228eec602 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-edfd2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-edfd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2.18c45fc690759c9a namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: db1cae36-0fce-464d-9708-2da2c0d165cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:57:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 49385531-e6f8-4d7e-85ad-dea228eec602 kind: Event lastTimestamp: "2026-07-21T17:57:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-edfd2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-21T17:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:57:38Z" name: success-200-isvc-edfd2.18c45fc69075e244 namespace: kserve-ci-e2e-test resourceVersion: "25938" uid: bf265505-a87a-46d6-8143-f48081a2cae5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-edfd2 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 49385531-e6f8-4d7e-85ad-dea228eec602 kind: Event lastTimestamp: "2026-07-21T17:58:26Z" message: InferenceService [success-200-isvc-edfd2] is Ready metadata: creationTimestamp: "2026-07-21T17:58:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:26Z" name: success-200-isvc-edfd2.18c45fd19fbf192e namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: 0a112ce2-7f19-4a04-a3a4-06f7461af7a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:47:03.535850Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20307" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f64bd-predictor-7764f488c-c4ttx to ip-10-0-132-249.ec2.internal metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32a91db6e4 namespace: kserve-ci-e2e-test resourceVersion: "20315" uid: b4e76bd7-a066-4333-b9ad-4a353ecb710c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f64bd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32b12c430c namespace: kserve-ci-e2e-test resourceVersion: "20346" uid: 6582a07b-a8e1-447d-93e8-ca05ab0e2354 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32e7425605 namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 5cfc8e6c-14f7-4a67-9b91-b8be5d6cae59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1775" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32e89578d5 namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: 8d506add-0669-4bf3-b8cb-b0dfde142e04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32ec55436a namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 076ae9ca-d2d5-40b3-8096-c84b8d6683e0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32ecd6bb49 namespace: kserve-ci-e2e-test resourceVersion: "20369" uid: 8ca63c20-f2c2-4cb4-b32c-226423e23067 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32ece1ee9a namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: c6c7dbb1-54ad-4bcb-a421-d81bccb6db03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32f03a837e namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 2b8d595a-9e38-453b-9a25-fcc02959c5c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:47:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:04Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f32f0b9b5e6 namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: 77db79c8-7474-4626-aab2-601578547b3a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:47:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:47:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:41Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f332bd9c288 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: a006a6b2-6367-4b6c-aca3-8b60ddcb7135 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f4303816954 namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: d14849e1-dedc-439e-97af-3697c40c56f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:48:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-21T17:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:13Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f430381dbd0 namespace: kserve-ci-e2e-test resourceVersion: "21209" uid: d0a77cfc-b2af-462e-8433-59133a36a1f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: 80b3aae2-8538-4e58-8249-c708ca4fde57 kind: Event lastTimestamp: "2026-07-21T17:48:16Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-21T17:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:48:16Z" name: success-200-isvc-f64bd-predictor-7764f488c-c4ttx.18c45f43b42cdc70 namespace: kserve-ci-e2e-test resourceVersion: "21312" uid: 29baf706-0dc8-4321-bbae-4ed023881efa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-132-249.ec2.internal source: component: kubelet host: ip-10-0-132-249.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f64bd-predictor-7764f488c namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 7f768887-0a8d-4cbe-9ea0-909da4b0915f kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: 'Created pod: success-200-isvc-f64bd-predictor-7764f488c-c4ttx' metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-f64bd-predictor-7764f488c.18c45f32a8b6dc45 namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: e6d5d03e-cbb3-450c-b124-c86581f2ce5a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20285" uid: be03c023-df02-4d4c-9aa6-720f8b0897cd kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Scaled up replica set success-200-isvc-f64bd-predictor-7764f488c from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:03Z" name: success-200-isvc-f64bd-predictor.18c45f32a53474e7 namespace: kserve-ci-e2e-test resourceVersion: "20288" uid: 65f5d233-9229-4661-81a1-db68f8b3ec0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: edcdd091-1c3e-41f8-a342-5120ecd831a1 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-f64bd-predictor.18c45f3624371f6d namespace: kserve-ci-e2e-test resourceVersion: "20676" uid: 34fee570-60a4-42df-9194-edd4d99e2866 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20286" uid: edcdd091-1c3e-41f8-a342-5120ecd831a1 kind: Event lastTimestamp: "2026-07-21T17:47:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:47:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:47:33Z" name: success-200-isvc-f64bd-predictor.18c45f36249d2dff namespace: kserve-ci-e2e-test resourceVersion: "20681" uid: 05827580-d488-4d4e-8961-e0fcbbbf9338 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: edcdd091-1c3e-41f8-a342-5120ecd831a1 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: success-200-isvc-f64bd-predictor.18c45f3d2276570a namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: b12eae7e-11ac-4a59-ad99-369ad3d7b393 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:47:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f64bd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: edcdd091-1c3e-41f8-a342-5120ecd831a1 kind: Event lastTimestamp: "2026-07-21T17:48:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-21T17:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:48:03Z" name: success-200-isvc-f64bd-predictor.18c45f3d22e3f111 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: ca51efa0-ac45-4305-9835-8187be23aabe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64bd namespace: kserve-ci-e2e-test resourceVersion: "20276" uid: 6799c10c-cff3-4f8d-81ca-3cddc76955ef kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:51Z" name: success-200-isvc-f64bd.18c45f32a536c04f namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: f529a4ca-2fab-42b7-861c-209b2ee3d8b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f64bd namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: 6799c10c-cff3-4f8d-81ca-3cddc76955ef kind: Event lastTimestamp: "2026-07-21T17:47:51Z" message: InferenceService [success-200-isvc-f64bd] is Ready metadata: creationTimestamp: "2026-07-21T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:47:51Z" name: success-200-isvc-f64bd.18c45f3de73daed4 namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: fbf584ce-ef59-40df-a30a-8bd6e01d16c0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:37:38.319265Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4dfe2-bb5dbc976-h6n8w to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eaf0f950027 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 244c8ead-e02f-4e5b-90b2-f418abb7430e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:38Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eaf29051f20 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 23707a37-395c-464e-9fe1-7dab8222e80d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eaf2a458e01 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: ec6762a5-7c15-41fe-8c49-e535784892d2 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" in 2.823s (2.823s including waiting). Image size: 218229850 bytes.' metadata: creationTimestamp: "2026-07-21T17:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:41Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eafd28bf4e9 namespace: kserve-ci-e2e-test resourceVersion: "15803" uid: 3e3337aa-33a2-4ae5-875c-77570adc7b2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:41Z" message: 'Created container: switch-graph-4dfe2' metadata: creationTimestamp: "2026-07-21T17:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:41Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eafd5b1f1b7 namespace: kserve-ci-e2e-test resourceVersion: "15804" uid: 06b1d7b6-ca35-4191-819e-f5e636da5927 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:41Z" message: Started container switch-graph-4dfe2 metadata: creationTimestamp: "2026-07-21T17:37:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:41Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eafd6256f42 namespace: kserve-ci-e2e-test resourceVersion: "15805" uid: c8cc0fb4-a1a6-43d2-b608-8fbd605c7e85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:37:48Z" message: Stopping container switch-graph-4dfe2 metadata: creationTimestamp: "2026-07-21T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:37:48Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eb16a516d94 namespace: kserve-ci-e2e-test resourceVersion: "15894" uid: 89c1ff63-e652-4031-8e62-5b2e1cb4f061 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dfe2} kind: Pod name: switch-graph-4dfe2-bb5dbc976-h6n8w namespace: kserve-ci-e2e-test resourceVersion: "15762" uid: 2b312ed3-829f-44a2-b234-94f65637c42d kind: Event lastTimestamp: "2026-07-21T17:38:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:18Z" name: switch-graph-4dfe2-bb5dbc976-h6n8w.18c45eb285a12fd8 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 9148996d-dcd6-46b7-a41a-95bd998cea6c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4dfe2-bb5dbc976 namespace: kserve-ci-e2e-test resourceVersion: "15756" uid: 8a1a1bd9-9610-495a-adf4-708aebf5f3fe kind: Event lastTimestamp: "2026-07-21T17:37:38Z" message: 'Created pod: switch-graph-4dfe2-bb5dbc976-h6n8w' metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2-bb5dbc976.18c45eaf0f1c67fa namespace: kserve-ci-e2e-test resourceVersion: "15763" uid: 57291efd-86f4-43f7-b573-afa0d8c1a932 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: e35c8b2d-80b9-4804-aa80-5cdc45d0e9de kind: Event lastTimestamp: "2026-07-21T17:37:38Z" message: Scaled up replica set switch-graph-4dfe2-bb5dbc976 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2.18c45eaf0e0da568 namespace: kserve-ci-e2e-test resourceVersion: "15757" uid: 4ff114f0-51c8-4163-9064-e6c3684fcf1a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:37:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4dfe2 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: 8a9cfb5e-ac09-4a1f-a6c4-517b4f45312e kind: Event lastTimestamp: "2026-07-21T17:37:38Z" message: InferenceGraph [switch-graph-4dfe2] is Ready metadata: creationTimestamp: "2026-07-21T17:37:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:37:38Z" name: switch-graph-4dfe2.18c45eaf14186783 namespace: kserve-ci-e2e-test resourceVersion: "15774" uid: 210bfec6-889e-43fa-9beb-7e2fb4870010 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:38:48.693642Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-5bf12-8c97ccc4-nrwbw to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:38:48Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebf7238c771 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: 20d92ff3-9f4f-4cf5-ad20-ed0484d12022 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-5bf12-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:48Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebf7c584fd1 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 9126922e-8611-43e9-8817-f626632e9a1f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:49Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16778" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:38:49Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:38:49Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebfaf8158cd namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: d0ea4d22-e100-4757-aafb-5d5d0f56642c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bf12} kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:38:49Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:49Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebfb0d4a3b9 namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: 37d5dbba-3cd6-47f6-a59f-9f2fd8472d3a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bf12} kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:38:49Z" message: 'Created container: switch-graph-5bf12' metadata: creationTimestamp: "2026-07-21T17:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:49Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebfb4151558 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 09dbc5dd-99f2-472e-9d5c-22d815cf5a5a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bf12} kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:38:49Z" message: Started container switch-graph-5bf12 metadata: creationTimestamp: "2026-07-21T17:38:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:38:49Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45ebfb48600d8 namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: d3849dea-be7a-4757-aa0c-14d6d3a0d8b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:47:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bf12} kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:47:03Z" message: Stopping container switch-graph-5bf12 metadata: creationTimestamp: "2026-07-21T17:47:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:03Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45f329a2ab7e5 namespace: kserve-ci-e2e-test resourceVersion: "20253" uid: 3c3cce66-ae82-4af7-9860-7432c0d0a478 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-21T17:47:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-5bf12} kind: Pod name: switch-graph-5bf12-8c97ccc4-nrwbw namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: fdc05d56-33fd-4cb8-a9bd-4d40bfada364 kind: Event lastTimestamp: "2026-07-21T17:47:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T17:47:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:47:31Z" name: switch-graph-5bf12-8c97ccc4-nrwbw.18c45f3359decb3c namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: efa7d2b4-e9e7-4d29-8f28-fd7707a538e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-5bf12-8c97ccc4 namespace: kserve-ci-e2e-test resourceVersion: "16763" uid: 78d66978-d016-4e2a-86ae-298f3b267d12 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: 'Created pod: switch-graph-5bf12-8c97ccc4-nrwbw' metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: switch-graph-5bf12-8c97ccc4.18c45ebf71b10fe7 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 901ee197-4e39-4bb2-8023-47f49190ff93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: 70e844e2-328c-47a8-9a1f-21b38df8b577 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: Scaled up replica set switch-graph-5bf12-8c97ccc4 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:38:48Z" name: switch-graph-5bf12.18c45ebf70e83f32 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 84b152af-b206-4800-addd-344d50cc103a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:38:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16780" uid: 81c40ce6-a689-4692-9759-f636793bd907 kind: Event lastTimestamp: "2026-07-21T17:38:48Z" message: InferenceGraph [switch-graph-5bf12] is Ready metadata: creationTimestamp: "2026-07-21T17:38:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:38:48Z" name: switch-graph-5bf12.18c45ebf765f5e2c namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 06a3d6af-5fac-46a8-918f-f07a3a1b4171 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 3140a5cb-94c5-4153-bf82-2f6ad9bd6848 kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:18Z" name: switch-graph-5bf12.18c45ec2ef60d265 namespace: kserve-ci-e2e-test resourceVersion: "17067" uid: ef1f42af-6460-450e-b601-e4b3b50ab06f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-21T17:39:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-5bf12 namespace: kserve-ci-e2e-test resourceVersion: "16762" uid: 3140a5cb-94c5-4153-bf82-2f6ad9bd6848 kind: Event lastTimestamp: "2026-07-21T17:39:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:39:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:39:18Z" name: switch-graph-5bf12.18c45ec2efd424ef namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 7546e3c5-0221-4d96-9d38-9c0328fe09bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-21T17:58:58.750829Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-984cf-684f6d5564-4wqx5 to ip-10-0-135-98.ec2.internal metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-21T17:58:58Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd92f402b56 namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: a494d483-fb2e-4ae2-b3f8-d9bd23e54ecf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5c6974dc94-dklk8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T17:58:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-984cf-serving-cert" not found' metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:58Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd93b151f31 namespace: kserve-ci-e2e-test resourceVersion: "27078" uid: c4b08025-7a52-4879-86ef-8ac42c6144f7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:59Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T17:58:59Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-21T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-21T17:58:59Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd96c998ea8 namespace: kserve-ci-e2e-test resourceVersion: "27085" uid: 11007931-2051-4b3f-bf32-d8640cd1a210 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-984cf} kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T17:58:59Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:4c5b35bc47640cedc469a58dc4db3d0dafd2fc41a58ddd3f20a70ce44e68325e" already present on machine metadata: creationTimestamp: "2026-07-21T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:59Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd96dee3d14 namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 5f1c3dd8-0a4d-4ee4-bca2-57cd1d8051fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-984cf} kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T17:58:59Z" message: 'Created container: switch-graph-984cf' metadata: creationTimestamp: "2026-07-21T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:59Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd9712f5a54 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: eedf5feb-a240-40cc-baac-77c940c07c6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-984cf} kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T17:58:59Z" message: Started container switch-graph-984cf metadata: creationTimestamp: "2026-07-21T17:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T17:58:59Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c45fd971a461bc namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 1aec313f-c315-4fcc-b9b2-f3872755cb8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T18:15:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-984cf} kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T18:15:18Z" message: Stopping container switch-graph-984cf metadata: creationTimestamp: "2026-07-21T18:15:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:18Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c460bd358d6cbc namespace: kserve-ci-e2e-test resourceVersion: "33966" uid: e1bd1a62-69f1-4d99-9be5-30a78795c963 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-21T18:15:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-984cf} kind: Pod name: switch-graph-984cf-684f6d5564-4wqx5 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: ee9a7e45-62a3-498c-ae89-6e4232cd3b60 kind: Event lastTimestamp: "2026-07-21T18:15:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-21T18:15:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-21T18:15:40Z" name: switch-graph-984cf-684f6d5564-4wqx5.18c460bddcd0911d namespace: kserve-ci-e2e-test resourceVersion: "34206" uid: ccab8b4f-bdb2-473f-8502-b8211df569a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-135-98.ec2.internal source: component: kubelet host: ip-10-0-135-98.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-984cf-684f6d5564 namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 5baac12c-b195-4a43-bc5a-ee8cb2be3ef5 kind: Event lastTimestamp: "2026-07-21T17:58:58Z" message: 'Created pod: switch-graph-984cf-684f6d5564-4wqx5' metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:58Z" name: switch-graph-984cf-684f6d5564.18c45fd92ee294b2 namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 45ef5545-1d25-4540-9953-8de01a7a8383 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-984cf namespace: kserve-ci-e2e-test resourceVersion: "27053" uid: 0b7af504-c9cb-4719-94c0-7b21b0fe5269 kind: Event lastTimestamp: "2026-07-21T17:58:58Z" message: Scaled up replica set switch-graph-984cf-684f6d5564 from 0 to 1 metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:58:58Z" name: switch-graph-984cf.18c45fd92e46012f namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: cc5ae7da-28ee-48f3-b249-b0a343ebf5a3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-21T17:58:58Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-984cf namespace: kserve-ci-e2e-test resourceVersion: "27069" uid: 45d30be2-73b4-426a-b168-26007055b63f kind: Event lastTimestamp: "2026-07-21T17:58:58Z" message: InferenceGraph [switch-graph-984cf] is Ready metadata: creationTimestamp: "2026-07-21T17:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-21T17:58:58Z" name: switch-graph-984cf.18c45fd93350a929 namespace: kserve-ci-e2e-test resourceVersion: "27073" uid: 371e81a4-3de5-48b6-abb8-f4b80abcdbd7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:59:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-984cf namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 3659dbfa-a872-40d4-922c-b0acac6a7b68 kind: Event lastTimestamp: "2026-07-21T17:59:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:43Z" name: switch-graph-984cf.18c45fdcaca5c823 namespace: kserve-ci-e2e-test resourceVersion: "27449" uid: e44a2b14-0537-4752-9a2d-783776e84679 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-21T17:59:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-984cf namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: 3659dbfa-a872-40d4-922c-b0acac6a7b68 kind: Event lastTimestamp: "2026-07-21T17:59:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-21T17:59:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-21T17:59:43Z" name: switch-graph-984cf.18c45fdcad02e024 namespace: kserve-ci-e2e-test resourceVersion: "27450" uid: 404b6032-6292-4136-9456-cc41bc586f53 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34243"