--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:32:15.686822Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21959" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-4e070-57b477589f-dnp8r to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:32:15Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66d88954d423e namespace: kserve-ci-e2e-test resourceVersion: "21964" uid: 4d8258f4-06f2-4d1c-96c4-ff90e469295a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:32:16Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66d88af548c3e namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 00a10250-d7e0-4568-8f88-5f2f7e2dbaae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4e070} kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66d88b12aa64a namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: fd5a653e-c8bf-433f-b02f-c18e67021907 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4e070} kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: 'Created container: ensemble-graph-4e070' metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66d88b4639db7 namespace: kserve-ci-e2e-test resourceVersion: "21995" uid: b46ab536-f819-4581-96d6-6c3963f39fac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4e070} kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Started container ensemble-graph-4e070 metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66d88b4dc35b0 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: 531d691f-66f1-4462-8950-d817a6a17027 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4e070} kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Stopping container ensemble-graph-4e070 metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66dfbc303dd5e namespace: kserve-ci-e2e-test resourceVersion: "25879" uid: 29ec8b15-016e-4301-a7d3-d798b9471d52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:40:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-4e070} kind: Pod name: ensemble-graph-4e070-57b477589f-dnp8r namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: f0b7062d-bf7e-449a-8cef-c4a14fffbd99 kind: Event lastTimestamp: "2026-07-28T10:40:57Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:57Z" name: ensemble-graph-4e070-57b477589f-dnp8r.18c66dfc44e4b056 namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: 53d942e3-6d07-4f11-bbdd-fbade7914d15 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-4e070-57b477589f namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: 01afbe08-5af1-4f5c-bd2e-2248120f45e3 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'Created pod: ensemble-graph-4e070-57b477589f-dnp8r' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: ensemble-graph-4e070-57b477589f.18c66d8894b5b1c6 namespace: kserve-ci-e2e-test resourceVersion: "21960" uid: cd0c9de0-0ae9-42e8-9b63-0d122987a833 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21951" uid: e3941411-2bdb-4e91-9a1a-e439e9b77229 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Scaled up replica set ensemble-graph-4e070-57b477589f from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: ensemble-graph-4e070.18c66d88940c1b6d namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: 3209d1a0-fb10-4199-ac5e-90c8c7c83855 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 93c414f2-56cf-4f8f-8c65-de3e32f3cdee kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: InferenceGraph [ensemble-graph-4e070] is Ready metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:15Z" name: ensemble-graph-4e070.18c66d88984e0c2d namespace: kserve-ci-e2e-test resourceVersion: "21976" uid: 518192bb-8819-4a47-8c34-a3e2c89a4e4c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 9a6bd06e-24e8-437a-b68e-a54e6abbe185 kind: Event lastTimestamp: "2026-07-28T10:32:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:30Z" name: ensemble-graph-4e070.18c66d8c12f0f9cd namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 7de07d0c-e4c7-4991-9f9b-24250f9fd10d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21954" uid: 9a6bd06e-24e8-437a-b68e-a54e6abbe185 kind: Event lastTimestamp: "2026-07-28T10:32:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:30Z" name: ensemble-graph-4e070.18c66d8c135ca2fe namespace: kserve-ci-e2e-test resourceVersion: "22157" uid: 96e605bf-7262-4131-961d-30854ba543e1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 9a6bd06e-24e8-437a-b68e-a54e6abbe185 kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: ensemble-graph-4e070.18c66d8f9208d7bb namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: 04069759-5829-445e-a2ff-6c768032b916 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-4e070 namespace: kserve-ci-e2e-test resourceVersion: "22156" uid: 9a6bd06e-24e8-437a-b68e-a54e6abbe185 kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: ensemble-graph-4e070.18c66d8f929ecf4f namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: fe246df1-77fc-4bbf-8409-72d44835e469 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:31:05.383277Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-af89d-db7874558-kldsc to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d7836e2512d namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 111a20e8-9f81-4156-a1ba-cf72d5bf9a3f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d7850f6c44b namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: 22328016-09ed-4e4d-8228-b3648971f30e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af89d} kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d78523b002a namespace: kserve-ci-e2e-test resourceVersion: "21060" uid: 35cd243e-cede-41de-b218-2514af791e66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af89d} kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'Created container: ensemble-graph-af89d' metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d78555a0f3e namespace: kserve-ci-e2e-test resourceVersion: "21063" uid: 3ab8fc66-ed28-4cb5-add0-e23bc60307a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af89d} kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: Started container ensemble-graph-af89d metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d7855d256c2 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 2cbbd3c2-b651-466f-b87f-962d6c59db5b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af89d} kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Stopping container ensemble-graph-af89d metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d7a8f141e80 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 863b04b5-ec6e-4d1f-abaa-0015bb02a64a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-af89d} kind: Pod name: ensemble-graph-af89d-db7874558-kldsc namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: e4d9dc0a-bc22-4fe6-b7c6-79a291a7b451 kind: Event lastTimestamp: "2026-07-28T10:31:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:42Z" name: ensemble-graph-af89d-db7874558-kldsc.18c66d7af8dcd2a4 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: e9611d4f-f09f-4b5a-b648-bf21ff0912e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-af89d-db7874558 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: b2e88efe-5f37-42a0-b269-d845c7edc334 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'Created pod: ensemble-graph-af89d-db7874558-kldsc' metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d-db7874558.18c66d783668375f namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: 136250a1-3cc9-46fe-b88d-19b1f7a125ee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-af89d namespace: kserve-ci-e2e-test resourceVersion: "21028" uid: 752a1eae-fb98-492f-a1bf-f4caaba97faa kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: Scaled up replica set ensemble-graph-af89d-db7874558 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d.18c66d7835b13758 namespace: kserve-ci-e2e-test resourceVersion: "21031" uid: b2f07147-c150-4d62-9388-0a3a10d3cf45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-af89d namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: e8225e10-7866-4ce3-ad86-31d8fcbaf586 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: InferenceGraph [ensemble-graph-af89d] is Ready metadata: creationTimestamp: "2026-07-28T10:31:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:05Z" name: ensemble-graph-af89d.18c66d783b65fdef namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 3e092f20-f620-4bb2-b51e-26d6ab82f1b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:21:40.555123Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:21:40Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf4b480a820 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 986dda0a-d272-4c4e-a370-4acedf884b59 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0b9d4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf4bb303c47 namespace: kserve-ci-e2e-test resourceVersion: "16564" uid: 8ebd237b-252c-4e28-a247-6e762b084e0b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf5274c9455 namespace: kserve-ci-e2e-test resourceVersion: "16590" uid: cc37668f-31c9-4847-bcf4-e53584bec8ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf528932338 namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: fb036afd-0447-443f-97d7-2781efec0152 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf52c9089f2 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 0f58a56e-ea17-4554-aaee-870e590ad46a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf52cf1b1c1 namespace: kserve-ci-e2e-test resourceVersion: "16594" uid: 902783fc-91ca-49e4-9532-359b616241d9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf52cfba7bf namespace: kserve-ci-e2e-test resourceVersion: "16595" uid: 39f7cfd6-a8ec-4b15-8dbb-c2bab024649f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf52f9fe882 namespace: kserve-ci-e2e-test resourceVersion: "16596" uid: 075fd0d5-abf6-464f-98b1-fab433484799 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf530001b25 namespace: kserve-ci-e2e-test resourceVersion: "16597" uid: f2d5c37e-11e8-46df-910b-0b641aa851cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:21:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:22:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:21:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:19Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66cf5805f9d58 namespace: kserve-ci-e2e-test resourceVersion: "16960" uid: ac85f72a-7d59-4f97-8885-746d3da87379 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66d75d63ba32e namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: c85d7806-67ed-40f2-ae8f-39fbe4008fca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5 namespace: kserve-ci-e2e-test resourceVersion: "16541" uid: 5493527c-3b35-4768-aa56-fbbcc9688dc3 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5.18c66d75d63c13c1 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 6ef65a79-7b8d-4243-b932-afc334977f68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0b9d4-predictor-7fdbfdf779 namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 021f23dc-6938-41ca-9a15-7ee35f546153 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: 'Created pod: error-404-isvc-0b9d4-predictor-7fdbfdf779-f5hq5' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:40Z" name: error-404-isvc-0b9d4-predictor-7fdbfdf779.18c66cf4b416268f namespace: kserve-ci-e2e-test resourceVersion: "16540" uid: a500df12-571e-432b-aa9b-e22a2b580f4f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16528" uid: 271500e8-d29e-4fcf-90f4-14969eb5c1bb kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Scaled up replica set error-404-isvc-0b9d4-predictor-7fdbfdf779 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:40Z" name: error-404-isvc-0b9d4-predictor.18c66cf4b2a32bf0 namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: bb17eecc-3b8f-47ee-9aea-bd038a00e472 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 4eea2949-4e49-49a1-8e99-af1fd889e553 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: error-404-isvc-0b9d4-predictor.18c66cf83132c2bc namespace: kserve-ci-e2e-test resourceVersion: "17156" uid: 2af687b8-5165-4911-a84d-c2ffd178f6df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 4eea2949-4e49-49a1-8e99-af1fd889e553 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: error-404-isvc-0b9d4-predictor.18c66cf831a0c622 namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 76ac42c6-e496-4c46-81ba-a3603c4621e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: 4eea2949-4e49-49a1-8e99-af1fd889e553 kind: Event lastTimestamp: "2026-07-28T10:22:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:10Z" name: error-404-isvc-0b9d4-predictor.18c66cfbb05c7d9e namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 97c120ae-5083-470e-826f-4187518e2a7d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: 4eea2949-4e49-49a1-8e99-af1fd889e553 kind: Event lastTimestamp: "2026-07-28T10:22:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:10Z" name: error-404-isvc-0b9d4-predictor.18c66cfbb0c5ce9a namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 985309ca-4544-4be2-be92-1b7cf6cf679a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 695f7032-6b4c-4574-a673-6d43334fab4c kind: Event lastTimestamp: "2026-07-28T10:22:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:22:30Z" name: error-404-isvc-0b9d4.18c66cf4b2b04fc5 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 19779c3e-4cc5-436a-a0b1-8a5ebad3b3ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 695f7032-6b4c-4574-a673-6d43334fab4c kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-0b9d4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0b9d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:40Z" name: error-404-isvc-0b9d4.18c66cf4b3292a91 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: e8f70d71-947c-42ac-ae0f-e8833ca0cf29 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "16511" uid: 695f7032-6b4c-4574-a673-6d43334fab4c kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-0b9d4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:40Z" name: error-404-isvc-0b9d4.18c66cf4b3296f9b namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: c2f4ff9e-f97c-4654-add1-a3c360af2482 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17058" uid: 695f7032-6b4c-4574-a673-6d43334fab4c kind: Event lastTimestamp: "2026-07-28T10:22:30Z" message: InferenceService [error-404-isvc-0b9d4] is Ready metadata: creationTimestamp: "2026-07-28T10:22:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:22:30Z" name: error-404-isvc-0b9d4.18c66d003bb07416 namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 10594500-14f9-49d1-b588-ca16bd668b8f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:30:55.293374Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d75dd7a9ff5 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: cdf104a1-d335-4bbd-93f8-e5b1acf54739 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-222a3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d75e56ee9db namespace: kserve-ci-e2e-test resourceVersion: "20904" uid: 01a91570-1387-4b21-b472-b774ac03e43b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20897" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d761d58f638 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: e0b77fe0-d556-486e-9880-32275f35fea8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d761ebc23c3 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 08b2a3fd-c8a6-4723-acb0-6b3e7512a2da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d762140120a namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: 4875e22c-411e-4ee3-b956-f9d3b192895e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d7621af5fda namespace: kserve-ci-e2e-test resourceVersion: "20931" uid: 86b7c371-7be8-4400-92b9-c5307b77c2ce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d7621ba4a58 namespace: kserve-ci-e2e-test resourceVersion: "20932" uid: f8aa8ae3-8b35-4891-9fe4-d70d0a1755ff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d7625195a92 namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: c1737f83-1b2c-4899-8698-430c25c5770b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d76258a198c namespace: kserve-ci-e2e-test resourceVersion: "20934" uid: 34ea5e6b-62b7-431a-bd46-39d80388526e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:31:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:33Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d7647e9aeb4 namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 5064736e-a8f5-4f8f-ad02-167c8af3ff04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d88850e4824 namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: d22c525f-728a-4430-a322-0729e968fbf0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d88850ebef5 namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 88718201-a461-4bbf-8ed3-c70b1a06fa25 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4 namespace: kserve-ci-e2e-test resourceVersion: "20890" uid: 6beb6ebc-7e48-4676-9361-992f85efd983 kind: Event lastTimestamp: "2026-07-28T10:32:18Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:18Z" name: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4.18c66d8924325de8 namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 5fab1580-5cee-4768-bb73-c68a9b19ad11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-222a3-predictor-bb95bb5f7 namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 07abeb57-08e3-4375-a430-4e113e97cb71 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'Created pod: error-404-isvc-222a3-predictor-bb95bb5f7-jbnk4' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3-predictor-bb95bb5f7.18c66d75dd09ea8d namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: 49fdbfef-46a0-49bc-b1f8-d5d7cbb2168b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: c049e994-4a80-4217-8999-2b0b6457570f kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Scaled up replica set error-404-isvc-222a3-predictor-bb95bb5f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3-predictor.18c66d75db243559 namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: cc7000ab-7adf-48c6-8029-7ca158d7544b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 6f5993f1-52ae-4c4f-a955-476e6d19f762 kind: Event lastTimestamp: "2026-07-28T10:31:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:55Z" name: error-404-isvc-222a3-predictor.18c66d79599850b1 namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 93297b11-acbb-456a-8ab8-6da6580a30bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: 6f5993f1-52ae-4c4f-a955-476e6d19f762 kind: Event lastTimestamp: "2026-07-28T10:31:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:55Z" name: error-404-isvc-222a3-predictor.18c66d795a0ec53c namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: 69c891c3-3ea0-4381-9b08-50f0dfda5210 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 6f5993f1-52ae-4c4f-a955-476e6d19f762 kind: Event lastTimestamp: "2026-07-28T10:31:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:25Z" name: error-404-isvc-222a3-predictor.18c66d7cd8be6495 namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 36c0d3ff-f8ee-4993-9dd7-8d8f40113052 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 6f5993f1-52ae-4c4f-a955-476e6d19f762 kind: Event lastTimestamp: "2026-07-28T10:31:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:25Z" name: error-404-isvc-222a3-predictor.18c66d7cd92d4a45 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: dabed7ab-138e-41f4-bed0-97cc11e3c744 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: d396318f-a6a2-4502-a136-26a1d9f8bc83 kind: Event lastTimestamp: "2026-07-28T10:31:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:43Z" name: error-404-isvc-222a3.18c66d75db3b9ab9 namespace: kserve-ci-e2e-test resourceVersion: "21532" uid: 818b47ca-24f4-47d4-9f8f-889f8b7cfcb9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: d396318f-a6a2-4502-a136-26a1d9f8bc83 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-222a3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-222a3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3.18c66d75dba5b241 namespace: kserve-ci-e2e-test resourceVersion: "20885" uid: 94d4a0cc-3a43-4fef-a05e-20e537b49402 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: d396318f-a6a2-4502-a136-26a1d9f8bc83 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-222a3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:55Z" name: error-404-isvc-222a3.18c66d75dba5fb75 namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: f23738b0-29a7-433d-bd31-e7b124ff1b72 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: d396318f-a6a2-4502-a136-26a1d9f8bc83 kind: Event lastTimestamp: "2026-07-28T10:31:43Z" message: InferenceService [error-404-isvc-222a3] is Ready metadata: creationTimestamp: "2026-07-28T10:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:43Z" name: error-404-isvc-222a3.18c66d810359b626 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: fd572655-cb25-483c-abf2-7c866f87df2a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:31:15.744046Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4e070-predictor-f797d89df-t5768 to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7aa06f144c namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: afc7c30f-0817-46ef-94fd-e356fc552750 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7aba451186 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: 6c28099d-109d-4a40-a098-872f10764042 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7abbbaa141 namespace: kserve-ci-e2e-test resourceVersion: "21264" uid: 7529cae8-9267-487f-b26e-b4f0c93e5243 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7abefbf10f namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: a7367de6-10d2-4353-8084-0bdb7b016fb3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7abf6eaec7 namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 5dfab05c-b886-410a-ac3e-a4b1637a2792 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7abf7a0b63 namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: ccaca5df-52df-4f7e-8279-1ec2ad54b0bd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7ac2677da7 namespace: kserve-ci-e2e-test resourceVersion: "21269" uid: 46fb7a5f-dbac-469e-9a85-a51678522c89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7ac2dde317 namespace: kserve-ci-e2e-test resourceVersion: "21271" uid: a9cfd4df-9054-4d79-9767-7b745ac280dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:40:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:33Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66d7af5a1c6ad namespace: kserve-ci-e2e-test resourceVersion: "26025" uid: e4d4d4a9-c196-46ba-9d62-4f86485dbed1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66dfbd48ee258 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: c41fb4cf-9490-41b7-a607-857c2bdc15e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66dfbd48f7cde namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: ae3aa72a-addc-4667-9f52-57edc1ec3bae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4e070-predictor-f797d89df-t5768 namespace: kserve-ci-e2e-test resourceVersion: "21243" uid: 493e083a-503f-41a3-a0b7-b7e03faedef9 kind: Event lastTimestamp: "2026-07-28T10:40:33Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:33Z" name: error-404-isvc-4e070-predictor-f797d89df-t5768.18c66dfc6a09aea8 namespace: kserve-ci-e2e-test resourceVersion: "26023" uid: a83fa637-b880-4fd7-be68-c7b50a743e10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4e070-predictor-f797d89df namespace: kserve-ci-e2e-test resourceVersion: "21234" uid: 9fd64832-e73e-4fee-902a-3688493f4cae kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: 'Created pod: error-404-isvc-4e070-predictor-f797d89df-t5768' metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-4e070-predictor-f797d89df.18c66d7a9ff6b56c namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 5553cf4c-2010-4834-8444-5cbce35b57f4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: ac8fe3ed-e519-44ac-8828-d0758bd0ec13 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Scaled up replica set error-404-isvc-4e070-predictor-f797d89df from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-4e070-predictor.18c66d7a9e4316fd namespace: kserve-ci-e2e-test resourceVersion: "21235" uid: 49227357-3b07-4867-bc36-dbfee7ec4441 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a37c3f44-c7a3-4484-95e6-eefaf002546a kind: Event lastTimestamp: "2026-07-28T10:32:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:00Z" name: error-404-isvc-4e070-predictor.18c66d7e1cc8a41b namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: 329ea923-7a3b-4e80-a1ba-93c0eac6c82b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: a37c3f44-c7a3-4484-95e6-eefaf002546a kind: Event lastTimestamp: "2026-07-28T10:32:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:00Z" name: error-404-isvc-4e070-predictor.18c66d7e1d3a38cf namespace: kserve-ci-e2e-test resourceVersion: "21679" uid: ef21ccc1-e64a-42e7-83c0-d676ac2460cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: a37c3f44-c7a3-4484-95e6-eefaf002546a kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-4e070-predictor.18c66d8899327976 namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: f16b4eeb-b7cb-41aa-9949-4017e9967c77 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: a37c3f44-c7a3-4484-95e6-eefaf002546a kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-4e070-predictor.18c66d88999a0487 namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: d339308b-de0d-4db9-80ce-e997804663ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: ba313f5c-d422-42f7-b88c-a0c418fa7240 kind: Event lastTimestamp: "2026-07-28T10:32:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:03Z" name: error-404-isvc-4e070.18c66d7a9e4e548c namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 56146769-cd95-461f-ad72-7795c8cf72ed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: ba313f5c-d422-42f7-b88c-a0c418fa7240 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-4e070": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4e070": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-4e070.18c66d7a9ee815f2 namespace: kserve-ci-e2e-test resourceVersion: "21238" uid: 11d56ee6-e078-4ab1-9f32-50584425c6c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21215" uid: ba313f5c-d422-42f7-b88c-a0c418fa7240 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4e070": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-4e070.18c66d7a9ee85131 namespace: kserve-ci-e2e-test resourceVersion: "21239" uid: 1aba4d37-d8aa-432a-99f6-34b7d2704a4a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: ba313f5c-d422-42f7-b88c-a0c418fa7240 kind: Event lastTimestamp: "2026-07-28T10:32:03Z" message: InferenceService [error-404-isvc-4e070] is Ready metadata: creationTimestamp: "2026-07-28T10:32:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:03Z" name: error-404-isvc-4e070.18c66d85b146b1f4 namespace: kserve-ci-e2e-test resourceVersion: "21707" uid: 93405bda-5e53-4027-b817-9b65db1dcfac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:19:20.866340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-717c3-predictor-67f5fb9455-8qnmw to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:19:20Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd42e6701bd namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 8ef4e001-5de0-489a-8e55-c1fda33f88da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:19:21Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd447fe1bbf namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: 4f6e5bb4-ab8f-460f-9f92-7b0946c05814 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd4495ec208 namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 178c46f3-88c1-42f7-921d-e7eabe30bb5e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" in 15.563s (15.563s including waiting). Image size: 1333940832 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd7e90d8aed namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: b4870acd-2ae4-4629-aeff-74d0ea24f7a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd7ec9c476a namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 87ff82a3-0c0d-4fbe-980d-71a06e3e7ebe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd7ed012bb9 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 4b0e5110-a821-427e-8337-17079675bf25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd7ed0f87af namespace: kserve-ci-e2e-test resourceVersion: "15110" uid: 5695bfa2-4d7d-4c24-8bff-ab69ea88740f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.446s (2.446s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd87edfdb07 namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: 8eb40e15-5a0c-4204-8f07-5928536c8ac8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd88208dd45 namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 85ee9fb9-ca28-43c6-af4d-87408e7b226e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd8826cb3bb namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: b5f9a297-2ef7-4a2b-9e48-a7287519d9e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:20:16Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:16Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66cd883caa78d namespace: kserve-ci-e2e-test resourceVersion: "15468" uid: 07277fe1-a39f-4a4a-9ffc-bdb75818ccf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66ce909719abc namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: c68e7db9-37b0-4823-a6db-d937dc58ae22 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66ce90972469d namespace: kserve-ci-e2e-test resourceVersion: "15871" uid: 31f4ce20-f27d-427a-8097-b96c9120771b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 96201bf8-4ea2-4c46-b607-c34fc7fc0fc6 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw.18c66ce947df4225 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: aa730242-b7ba-4830-a5f5-66589d4c40b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-717c3-predictor-67f5fb9455 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: a4ec02a0-b150-4ab5-b5bb-86684c2d75c8 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created pod: error-404-isvc-717c3-predictor-67f5fb9455-8qnmw' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: error-404-isvc-717c3-predictor-67f5fb9455.18c66cd42db3daa1 namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: d163a87b-a069-40e6-80de-c6c71a66a6b4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: 3b1ee574-86fa-4177-9d42-9232d0c27d04 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Scaled up replica set error-404-isvc-717c3-predictor-67f5fb9455 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: error-404-isvc-717c3-predictor.18c66cd42710d460 namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 3d28eefe-c4eb-4b46-8736-2a5a5cf72000 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: dc1b06d9-810a-4624-b3ab-20eec7e74bea kind: Event lastTimestamp: "2026-07-28T10:20:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:20Z" name: error-404-isvc-717c3-predictor.18c66cd7a5b8a62f namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: aae37763-a076-48e9-91a9-4424a8f30d22 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: dc1b06d9-810a-4624-b3ab-20eec7e74bea kind: Event lastTimestamp: "2026-07-28T10:20:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:20Z" name: error-404-isvc-717c3-predictor.18c66cd7a6608309 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 4c4d6b5c-a5bc-4ee5-819e-62111bcf4985 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:20:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: dc1b06d9-810a-4624-b3ab-20eec7e74bea kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: error-404-isvc-717c3-predictor.18c66cdea3fae955 namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: 27ff22be-cffd-4ffc-90de-f6ea710f04f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:20:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: dc1b06d9-810a-4624-b3ab-20eec7e74bea kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: error-404-isvc-717c3-predictor.18c66cdea4ae3702 namespace: kserve-ci-e2e-test resourceVersion: "15643" uid: b21e61ab-dbf4-4f09-bfae-6a3d6e0f24bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-717c3 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 1a4561eb-9f7f-4c6b-9391-25742333b969 kind: Event lastTimestamp: "2026-07-28T10:20:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:26Z" name: error-404-isvc-717c3.18c66cd4274a1136 namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: a0621104-5569-41e4-8a3d-52219a9bdcbe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: 1a4561eb-9f7f-4c6b-9391-25742333b969 kind: Event lastTimestamp: "2026-07-28T10:20:26Z" message: InferenceService [error-404-isvc-717c3] is Ready metadata: creationTimestamp: "2026-07-28T10:20:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:26Z" name: error-404-isvc-717c3.18c66ce37b2f5574 namespace: kserve-ci-e2e-test resourceVersion: "15566" uid: 32aa25a2-5a42-484f-a3e1-eb9d6de906c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:32:15.589807Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21933" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d888f84f3bc namespace: kserve-ci-e2e-test resourceVersion: "21937" uid: 2bc11184-d16e-4356-924d-29483e3ebaa3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8d409-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88998e7d15 namespace: kserve-ci-e2e-test resourceVersion: "21981" uid: 98e5acab-9538-41cb-b782-55a41cdd5a4d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21945" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88cd7a495d namespace: kserve-ci-e2e-test resourceVersion: "22006" uid: 13fd3a18-f2ab-4d12-8f1b-2aef76e13f48 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88cf48348d namespace: kserve-ci-e2e-test resourceVersion: "22009" uid: 36ff704e-b687-473c-89a4-9a9c470a034d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88d1604b2b namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: 2db9f3c9-ef11-41bb-9f73-c9fdbd6afa95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88d1e3b078 namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: e48217dd-f36d-419b-bad3-632d038100f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88d1ed4994 namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: c68b1d30-ecac-4f01-b154-731c4aac3230 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88d4867949 namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: 7d6c4c60-8c51-4e85-852a-a0cf2fc6d8d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88d501433a namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: 8bec5d01-22c1-4234-809f-12fdac1d4387 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:41:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:33Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66d88fc456aad namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: 012680d0-2b4f-416e-b06e-437faa3d6049 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66e09b25be642 namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: a50f0ea4-3c6f-47fa-9cc0-2448bd4e1c62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66e09b25c6466 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: 81e0f8d8-2a92-4caa-bb7e-f7beaa3c2087 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: a8da85ee-464a-4e98-87cf-7175cc847352 kind: Event lastTimestamp: "2026-07-28T10:41:33Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:33Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j.18c66e0a70c6b33c namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: d1f66ee3-0716-49df-be8d-17003aff35e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8d409-predictor-5fcc67fb8c namespace: kserve-ci-e2e-test resourceVersion: "21926" uid: 428921ea-9269-4660-b17b-4ab7d11978ac kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'Created pod: error-404-isvc-8d409-predictor-5fcc67fb8c-l8z5j' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409-predictor-5fcc67fb8c.18c66d888f10b721 namespace: kserve-ci-e2e-test resourceVersion: "21934" uid: fa913f60-1470-4d06-a196-ff715befb0aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: f03bb291-13cc-4c3a-a8eb-9b63eab5539b kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Scaled up replica set error-404-isvc-8d409-predictor-5fcc67fb8c from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409-predictor.18c66d888d840db0 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 92b233d5-02d9-408d-8eac-52132c74aaef reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 3d0a5bae-7e22-4552-beab-c4fe70b392ec kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: error-404-isvc-8d409-predictor.18c66d8c0bf1fee8 namespace: kserve-ci-e2e-test resourceVersion: "22267" uid: 2f8aca1c-89f4-4930-a525-39f341a55d76 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 3d0a5bae-7e22-4552-beab-c4fe70b392ec kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: error-404-isvc-8d409-predictor.18c66d8c0c4e3c07 namespace: kserve-ci-e2e-test resourceVersion: "22268" uid: 97d264fb-7205-4e3f-b2b1-967450856db1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:33:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "22149" uid: 3d0a5bae-7e22-4552-beab-c4fe70b392ec kind: Event lastTimestamp: "2026-07-28T10:33:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:33:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:15Z" name: error-404-isvc-8d409-predictor.18c66d9309bff2cb namespace: kserve-ci-e2e-test resourceVersion: "22533" uid: 4c820b21-78da-4c96-bb74-b4444b8b35bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:33:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "22149" uid: 3d0a5bae-7e22-4552-beab-c4fe70b392ec kind: Event lastTimestamp: "2026-07-28T10:33:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:33:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:15Z" name: error-404-isvc-8d409-predictor.18c66d930a2caf62 namespace: kserve-ci-e2e-test resourceVersion: "22535" uid: 8ad29a5c-f6ea-4158-828d-875bccd92436 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 77070441-27d4-4c4a-ba6a-49382105939b kind: Event lastTimestamp: "2026-07-28T10:33:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:33:03Z" name: error-404-isvc-8d409.18c66d888d840c1d namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 30ea3b05-456c-4ef0-b585-ea9caeba00de reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 77070441-27d4-4c4a-ba6a-49382105939b kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-8d409": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d409": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409.18c66d888e07995f namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: 35b38304-c131-406b-950b-2053860c8d27 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 77070441-27d4-4c4a-ba6a-49382105939b kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8d409": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:15Z" name: error-404-isvc-8d409.18c66d888e07d6e4 namespace: kserve-ci-e2e-test resourceVersion: "21931" uid: f33e4290-0bca-4528-a5a2-4bdd9683454c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22419" uid: 77070441-27d4-4c4a-ba6a-49382105939b kind: Event lastTimestamp: "2026-07-28T10:33:03Z" message: InferenceService [error-404-isvc-8d409] is Ready metadata: creationTimestamp: "2026-07-28T10:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:33:03Z" name: error-404-isvc-8d409.18c66d93bab90fab namespace: kserve-ci-e2e-test resourceVersion: "22420" uid: 9cec8e95-a1f9-49d4-a282-f33bbc9550e5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:41:30.317907Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26576" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a647d-predictor-784bd7c46f-s7brw to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09b7e3fdab namespace: kserve-ci-e2e-test resourceVersion: "26580" uid: 139ef626-0563-4fc5-b62b-e6f767127cd4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-a647d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09c30b7f10 namespace: kserve-ci-e2e-test resourceVersion: "26593" uid: d30c423f-6acd-4d43-ad54-47d7f60c7690 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09f4e2cde0 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: d2f8be1b-09f3-40f9-9f27-2ef4d93a45f1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09f639e386 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: d79f10a3-a5b4-418d-9063-4386b0d8ed2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09f9a63a5f namespace: kserve-ci-e2e-test resourceVersion: "26652" uid: 6d1057d9-a858-4c8d-8bf2-7ce6d066f2e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09fa16b499 namespace: kserve-ci-e2e-test resourceVersion: "26653" uid: 2458ade8-0a50-4c40-946e-4336152a37c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09fa204db9 namespace: kserve-ci-e2e-test resourceVersion: "26654" uid: 7886d3f0-f8b7-4996-bc06-6183f63b8cf2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09fccc0f2a namespace: kserve-ci-e2e-test resourceVersion: "26656" uid: 6ef18610-f6cc-4454-b02d-bb9f5692888a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09fd4f8092 namespace: kserve-ci-e2e-test resourceVersion: "26657" uid: b094a95b-d307-4357-8c18-83cc033db661 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:42:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:07Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66e09fe437087 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: 679a5d6a-3baa-431a-9a2d-323ca255ca9c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:58:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:49Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66efbb28bd132 namespace: kserve-ci-e2e-test resourceVersion: "34322" uid: 73dc1e99-02dc-4272-b849-7a4920a1f0e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:58:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:49Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66efbb28d1e17 namespace: kserve-ci-e2e-test resourceVersion: "34323" uid: e7b33e6b-6c8f-4045-bc0b-a8353606cbe4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw namespace: kserve-ci-e2e-test resourceVersion: "26578" uid: 475ccac3-d5d6-4509-9c8d-60f7801a8f2e kind: Event lastTimestamp: "2026-07-28T10:58:52Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:52Z" name: error-404-isvc-a647d-predictor-784bd7c46f-s7brw.18c66efc5f064897 namespace: kserve-ci-e2e-test resourceVersion: "34382" uid: 2c954ddb-638d-42e9-9df3-d5c89b2091a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a647d-predictor-784bd7c46f namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 6cddded0-7259-4610-9fbe-e7cece98bb68 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Created pod: error-404-isvc-a647d-predictor-784bd7c46f-s7brw' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d-predictor-784bd7c46f.18c66e09b76d7b2e namespace: kserve-ci-e2e-test resourceVersion: "26577" uid: eed9fb9c-1df0-4c24-bbdd-1f7d9887f3fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 934819d5-0ff6-4638-bec6-d745740b3c6d kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Scaled up replica set error-404-isvc-a647d-predictor-784bd7c46f from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d-predictor.18c66e09b5c4ccc6 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 54fc3658-666c-4973-880a-f21a59a2b551 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 231ad2c9-6285-41b5-bc86-8cc2fb474806 kind: Event lastTimestamp: "2026-07-28T10:42:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:00Z" name: error-404-isvc-a647d-predictor.18c66e0d3441bb39 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 81eba9d5-46ab-4c85-ad66-957ab8a49ea4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: 231ad2c9-6285-41b5-bc86-8cc2fb474806 kind: Event lastTimestamp: "2026-07-28T10:42:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:00Z" name: error-404-isvc-a647d-predictor.18c66e0d3494c8fe namespace: kserve-ci-e2e-test resourceVersion: "27068" uid: 0dee3b0f-da43-4b2e-a425-571f02123306 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26932" uid: 231ad2c9-6285-41b5-bc86-8cc2fb474806 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:30Z" name: error-404-isvc-a647d-predictor.18c66e1431dcfab7 namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: d00cd31b-21e0-4923-81f1-67ba4907f608 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26932" uid: 231ad2c9-6285-41b5-bc86-8cc2fb474806 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:30Z" name: error-404-isvc-a647d-predictor.18c66e1432467c87 namespace: kserve-ci-e2e-test resourceVersion: "27356" uid: ea7e3ec0-e069-4442-99ca-6412f16e8ee4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 3c35c8cc-3c83-4a99-ac73-629750d167bc kind: Event lastTimestamp: "2026-07-28T10:42:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:17Z" name: error-404-isvc-a647d.18c66e09b5cb38c3 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 23218b01-9b96-4a83-bc23-ac4f14cabc3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 3c35c8cc-3c83-4a99-ac73-629750d167bc kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-a647d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a647d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d.18c66e09b68efcd6 namespace: kserve-ci-e2e-test resourceVersion: "26574" uid: e39f13c4-c00f-46ca-af11-f52a4b888c99 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 3c35c8cc-3c83-4a99-ac73-629750d167bc kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a647d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-a647d.18c66e09b68f4422 namespace: kserve-ci-e2e-test resourceVersion: "26575" uid: 4d1c0315-382d-48eb-8d2b-b24e72d70a98 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "27207" uid: 3c35c8cc-3c83-4a99-ac73-629750d167bc kind: Event lastTimestamp: "2026-07-28T10:42:17Z" message: InferenceService [error-404-isvc-a647d] is Ready metadata: creationTimestamp: "2026-07-28T10:42:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:17Z" name: error-404-isvc-a647d.18c66e14ba25d15c namespace: kserve-ci-e2e-test resourceVersion: "27208" uid: 21396c05-5845-4800-a1a1-46e36ff6cc37 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:30:05.457268Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-af89d-predictor-6bff89d85-j66sp to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a4303fc1a namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 13c205d5-dbe7-4404-b530-c1ffd5d87b34 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a5cd78dce namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 95d58d88-d5a0-4d50-b931-455145a3ac6a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a5e2eec92 namespace: kserve-ci-e2e-test resourceVersion: "20375" uid: 324428ae-7ca7-48f0-a5d7-bec91574a8ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a6119aad4 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 181d0dc1-479d-4915-aecf-e37e59df1f25 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a618e3c21 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: bba276c8-6b6b-4416-a81b-182b0ce8b044 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a6198ab27 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 7aceace6-81b8-47a6-91e2-39fcb27b7689 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:06Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a645fecdb namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: aa352884-f70c-4fff-91a6-963a4fc6d479 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:06Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a64d3637b namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: da7df39b-bde0-494f-b650-4bfb8b84ca62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:30:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:42Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d6a96eb69cc namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: 1bd7a982-fae1-45a5-83db-91cce6af0d25 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d7a99dcd696 namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 3e6a241f-8352-45d4-b7cb-3a0d044f4340 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d7a99de1443 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: 8f9e29f5-4bd1-42fc-890b-6a31f8dbdb57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-af89d-predictor-6bff89d85-j66sp namespace: kserve-ci-e2e-test resourceVersion: "20336" uid: a8416209-8c09-4f1c-a627-65de821b7f0c kind: Event lastTimestamp: "2026-07-28T10:31:17Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:17Z" name: error-404-isvc-af89d-predictor-6bff89d85-j66sp.18c66d7b1f168dd3 namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: 6a9ecc50-86ca-45bb-927d-ef346e32df56 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-af89d-predictor-6bff89d85 namespace: kserve-ci-e2e-test resourceVersion: "20327" uid: 3164ce0a-8bb5-4410-a4e0-9d0a7f3d9d4d kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: 'Created pod: error-404-isvc-af89d-predictor-6bff89d85-j66sp' metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor-6bff89d85.18c66d6a429683ca namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: beacc09a-8591-480b-97cd-a386ba02b85f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: d8d18bf4-4528-4c6c-9450-8cece17da151 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Scaled up replica set error-404-isvc-af89d-predictor-6bff89d85 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-af89d-predictor.18c66d6a40da333c namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 1ce3941c-6cc1-4214-be18-916a8a4ce6e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:30:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 83ed6a28-b248-437a-8a24-22b4bad9715b kind: Event lastTimestamp: "2026-07-28T10:30:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:50Z" name: error-404-isvc-af89d-predictor.18c66d6dbf6c93b8 namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: a716979d-770b-4cb1-8dbf-ab860d822a1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:30:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20326" uid: 83ed6a28-b248-437a-8a24-22b4bad9715b kind: Event lastTimestamp: "2026-07-28T10:30:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:50Z" name: error-404-isvc-af89d-predictor.18c66d6dc0079a3b namespace: kserve-ci-e2e-test resourceVersion: "20742" uid: 1eaed16f-83f3-4085-8344-aa8363127929 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 83ed6a28-b248-437a-8a24-22b4bad9715b kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: error-404-isvc-af89d-predictor.18c66d713eae842c namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: ba9769c7-d533-497b-8648-80810656d0e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: 83ed6a28-b248-437a-8a24-22b4bad9715b kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: error-404-isvc-af89d-predictor.18c66d713f257bd0 namespace: kserve-ci-e2e-test resourceVersion: "21052" uid: 57f8d037-38d0-43e1-98d7-612b274b442d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af89d namespace: kserve-ci-e2e-test resourceVersion: "20308" uid: 9bcdb903-b574-49e4-b191-1c3ec5550958 kind: Event lastTimestamp: "2026-07-28T10:30:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:53Z" name: error-404-isvc-af89d.18c66d6a40e4e10b namespace: kserve-ci-e2e-test resourceVersion: "20777" uid: 545c4396-03a9-4d54-98f5-d858e9ba0338 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-af89d namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 9bcdb903-b574-49e4-b191-1c3ec5550958 kind: Event lastTimestamp: "2026-07-28T10:30:52Z" message: InferenceService [error-404-isvc-af89d] is Ready metadata: creationTimestamp: "2026-07-28T10:30:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:52Z" name: error-404-isvc-af89d.18c66d755274069a namespace: kserve-ci-e2e-test resourceVersion: "20771" uid: 7bb66377-c57b-48df-ab5e-e0dc7999cea6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:40:30.677077Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e803a-predictor-76bd58685f-h5jzr to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfbd5052b71 namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 3e42e3c7-3746-4328-81f6-b867841baa2e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e803a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfbdf9865a6 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: d3f21f44-4de7-4cc3-9310-9aa3d71eb40e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc13033c7f namespace: kserve-ci-e2e-test resourceVersion: "25998" uid: 7d559592-230e-48ae-8ba7-4bf52e4c33ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc146193ca namespace: kserve-ci-e2e-test resourceVersion: "26001" uid: 5198dc84-1f02-44a5-838c-360b4bc9041c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc17152eda namespace: kserve-ci-e2e-test resourceVersion: "26003" uid: b2a924e7-a876-40c3-b1d9-227527b21dc9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc17818430 namespace: kserve-ci-e2e-test resourceVersion: "26004" uid: 20296cd7-70e8-4646-bc6d-556590337c12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc178a5093 namespace: kserve-ci-e2e-test resourceVersion: "26005" uid: c1ee3754-1bda-4d33-86ea-77d384f4b604 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc1a388a24 namespace: kserve-ci-e2e-test resourceVersion: "26006" uid: 12f4ea36-83ca-48cd-8dae-1324b7e905c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc1b405f32 namespace: kserve-ci-e2e-test resourceVersion: "26008" uid: c975f1cc-d3c2-4774-afb1-f602c74a09d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:41:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:09Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66dfc6f86c478 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: 83f51a49-6eaf-4d61-8ca7-c75ebe300230 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66e0c2b29be57 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: 93ce245e-84cd-432a-8b15-cf08aa2f4d2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66e0c2b2c9fee namespace: kserve-ci-e2e-test resourceVersion: "26820" uid: f7368c1c-5015-464a-af42-99cf2a34f0a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: a61fa5c3-b09f-4404-bac7-3cd38f5f0924 kind: Event lastTimestamp: "2026-07-28T10:41:44Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:44Z" name: error-404-isvc-e803a-predictor-76bd58685f-h5jzr.18c66e0cf7da49a1 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 967f225a-3625-4c25-a52e-802d3fcd0291 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e803a-predictor-76bd58685f namespace: kserve-ci-e2e-test resourceVersion: "25936" uid: 973612f7-9013-4407-9813-f71359229334 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'Created pod: error-404-isvc-e803a-predictor-76bd58685f-h5jzr' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-e803a-predictor-76bd58685f.18c66dfbd454ea77 namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 0fc67f45-c93f-4e18-9f51-9d50dbf6429f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25933" uid: 7eeb427f-3b2a-4321-9d55-50acdae02b11 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Scaled up replica set error-404-isvc-e803a-predictor-76bd58685f from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:40:30Z" name: error-404-isvc-e803a-predictor.18c66dfbd27d5db1 namespace: kserve-ci-e2e-test resourceVersion: "25941" uid: d6f368bc-edcc-42eb-9164-d4023cd6b171 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:40:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: c00eb79a-da75-4d42-9c07-8d6c71417a4f kind: Event lastTimestamp: "2026-07-28T10:41:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:00Z" name: error-404-isvc-e803a-predictor.18c66dff50dcef09 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1dc90bed-219c-4d46-b754-788358d9ccec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:40:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25935" uid: c00eb79a-da75-4d42-9c07-8d6c71417a4f kind: Event lastTimestamp: "2026-07-28T10:41:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:00Z" name: error-404-isvc-e803a-predictor.18c66dff51472b60 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: fced3cd9-79dc-4973-9abe-5b704bfdcb28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: c00eb79a-da75-4d42-9c07-8d6c71417a4f kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-e803a-predictor.18c66e064ec63e4b namespace: kserve-ci-e2e-test resourceVersion: "26628" uid: 260eae77-27ef-493e-953d-0f870dea37bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: c00eb79a-da75-4d42-9c07-8d6c71417a4f kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: error-404-isvc-e803a-predictor.18c66e064f2ef355 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: 0ae5fd13-aca9-4a2c-8ae9-d6f1526b33c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "25919" uid: 23a6c97b-57e6-48e0-bb4b-f398b096caec kind: Event lastTimestamp: "2026-07-28T10:41:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:19Z" name: error-404-isvc-e803a.18c66dfbd274c6a4 namespace: kserve-ci-e2e-test resourceVersion: "26406" uid: 17338686-63f9-42bc-bf24-a8d0e8dec280 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: 23a6c97b-57e6-48e0-bb4b-f398b096caec kind: Event lastTimestamp: "2026-07-28T10:41:19Z" message: InferenceService [error-404-isvc-e803a] is Ready metadata: creationTimestamp: "2026-07-28T10:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:19Z" name: error-404-isvc-e803a.18c66e072e36930c namespace: kserve-ci-e2e-test resourceVersion: "26402" uid: 7ed2b31b-4d0f-440a-a32e-a5ac7fde43a2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:20:50.544435Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15896" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e92a5-predictor-599bf66b77-hrnbd to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce90fa221f6 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: fe1426dd-021c-4b61-851f-43cefef03ccb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce9291b8e90 namespace: kserve-ci-e2e-test resourceVersion: "15925" uid: 49844e1f-67b0-4e9e-90aa-783ad05ec574 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce92a68fa68 namespace: kserve-ci-e2e-test resourceVersion: "15927" uid: 7b36c8b2-7e6f-4a63-80c0-67b068ae1458 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce92d9cb4e1 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 7d546170-f2ec-4aa3-93b3-a1b4a85d7ca9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce92dff9a95 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: b148afc9-5b23-4f67-b3ab-810d0c4b7135 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce92e088d66 namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 2d171999-6599-4151-bb4d-211d142cfb64 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce930419019 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 877b6dcd-8b8a-4923-bfd4-3a5d99182d0e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce930c53069 namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 9c0aa980-7b99-48ec-893a-b2373c22174f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:30:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:07Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66ce957c54cb9 namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: a4b912fb-621e-4743-8b93-fb20ef3163f8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66d6a460cbdfd namespace: kserve-ci-e2e-test resourceVersion: "20356" uid: f1bf7894-85a8-4abb-8187-59e8d52a9787 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66d6a4610cad4 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 6fbfcd02-b9e6-44af-ac54-3ce2c446adf9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 98f5654d-0165-47e1-a1a1-ea27785211d5 kind: Event lastTimestamp: "2026-07-28T10:30:07Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:07Z" name: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd.18c66d6acc2458a4 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 9eb177b4-e971-476c-9c35-916692c09b91 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e92a5-predictor-599bf66b77 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 32de8e75-60ac-437f-b42f-5c6a3189b3e6 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'Created pod: error-404-isvc-e92a5-predictor-599bf66b77-hrnbd' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5-predictor-599bf66b77.18c66ce90f119b39 namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 2ff9b0c9-b2a8-4a9a-b846-b445062ddb04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15885" uid: 0b84c7e7-0af3-46b8-af84-3cef1a2db607 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Scaled up replica set error-404-isvc-e92a5-predictor-599bf66b77 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5-predictor.18c66ce90d7a33fa namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 03ca83f7-74a1-4818-9c7a-7b36e4ea5541 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: ec6abc92-9a74-45cb-b20c-4db3bc7c90ee kind: Event lastTimestamp: "2026-07-28T10:21:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:35Z" name: error-404-isvc-e92a5-predictor.18c66cec8c0dfb84 namespace: kserve-ci-e2e-test resourceVersion: "16374" uid: 272efa3a-6c35-42d9-9999-26ce447838f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: ec6abc92-9a74-45cb-b20c-4db3bc7c90ee kind: Event lastTimestamp: "2026-07-28T10:21:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:35Z" name: error-404-isvc-e92a5-predictor.18c66cec8c704578 namespace: kserve-ci-e2e-test resourceVersion: "16375" uid: fc98e755-da1e-4c64-8067-c8ba3ff89c72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: ec6abc92-9a74-45cb-b20c-4db3bc7c90ee kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: error-404-isvc-e92a5-predictor.18c66cf708bb94c0 namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 8782c448-b7fb-4142-bc28-fcc00fbf66c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: ec6abc92-9a74-45cb-b20c-4db3bc7c90ee kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: error-404-isvc-e92a5-predictor.18c66cf709269923 namespace: kserve-ci-e2e-test resourceVersion: "16853" uid: 4a431382-c756-4c78-b1f9-1bc08b1cfc50 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 59ff2649-150b-4a74-bc98-5f7ad912c727 kind: Event lastTimestamp: "2026-07-28T10:21:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:37Z" name: error-404-isvc-e92a5.18c66ce90da86223 namespace: kserve-ci-e2e-test resourceVersion: "16403" uid: db432086-76da-48d0-b897-1e17275c3060 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 59ff2649-150b-4a74-bc98-5f7ad912c727 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'Failed to update status for InferenceService "error-404-isvc-e92a5": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e92a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5.18c66ce90e1e93b3 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 5aedf3ac-c52e-4196-a63e-d9019f3eed55 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: 59ff2649-150b-4a74-bc98-5f7ad912c727 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e92a5": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:50Z" name: error-404-isvc-e92a5.18c66ce90e1ec88b namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: 7f8793ec-d6c7-4727-ac62-778e39b83259 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16400" uid: 59ff2649-150b-4a74-bc98-5f7ad912c727 kind: Event lastTimestamp: "2026-07-28T10:21:37Z" message: InferenceService [error-404-isvc-e92a5] is Ready metadata: creationTimestamp: "2026-07-28T10:21:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:37Z" name: error-404-isvc-e92a5.18c66cf4134abf2b namespace: kserve-ci-e2e-test resourceVersion: "16401" uid: f9277c8b-7ae3-49d8-9d3b-db49b5cdfb5b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:41:40.932500Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0c3091de7e namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 74f6b833-b3fa-4049-9557-7ee6d03f7d3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f7711-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:41Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0c3a593597 namespace: kserve-ci-e2e-test resourceVersion: "26867" uid: 953fc746-6e4d-4853-a6ab-36d8fe791280 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:41:42Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0ca3fc4993 namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: 1abfe635-266b-425d-9038-37f5d8135c65 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0ca57bb7e8 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 16633a74-6a76-4099-9640-7b35ed8588ab reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0ca84b9746 namespace: kserve-ci-e2e-test resourceVersion: "26889" uid: 4d41b5d5-72a3-469a-aba3-bc9cd3a65a4e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0ca8fc309d namespace: kserve-ci-e2e-test resourceVersion: "26890" uid: cc0b4fdc-5e56-4a32-8a08-34262c028c41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0ca9068577 namespace: kserve-ci-e2e-test resourceVersion: "26891" uid: 888a6ada-a1bc-4fd0-b051-6ee8019be582 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:43Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0cabd7941d namespace: kserve-ci-e2e-test resourceVersion: "26892" uid: 6fbcb053-762c-4ed1-812c-0038c8c40817 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:41:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:43Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0cac4478ba namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: bce24e42-5291-444b-80c9-0e9da8624f93 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:42:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:20Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e0d09526d91 namespace: kserve-ci-e2e-test resourceVersion: "27245" uid: ce125ae7-2cb1-42b9-81e6-16226d5d7510 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:50:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:50:55Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e8d623f99da namespace: kserve-ci-e2e-test resourceVersion: "31005" uid: d4822ede-d15d-4270-9147-1e958636cf9d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: 8c320d1a-ea74-423e-b59b-21aaff97675f kind: Event lastTimestamp: "2026-07-28T10:50:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:50:55Z" name: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk.18c66e8d6240ccb3 namespace: kserve-ci-e2e-test resourceVersion: "31006" uid: 1b5442b2-463c-47c1-8d08-38b60c14c913 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7711-predictor-6d447c4c9c namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 1559dce0-ea6c-4576-bf7c-f3b344b117a7 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: 'Created pod: error-404-isvc-f7711-predictor-6d447c4c9c-vwjvk' metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-f7711-predictor-6d447c4c9c.18c66e0c3020aa4c namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: 5932bc39-7b7e-480a-8649-fd7772a9e4fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: e9e3e3df-be6c-4327-8050-805765578978 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Scaled up replica set error-404-isvc-f7711-predictor-6d447c4c9c from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-f7711-predictor.18c66e0c2e7fe128 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: 02144b37-7a65-4163-9606-6950fa667826 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:41:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: d6089f19-7a21-4543-9e4c-5ea91e9a5bb1 kind: Event lastTimestamp: "2026-07-28T10:42:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:55Z" name: error-404-isvc-f7711-predictor.18c66e0facea6770 namespace: kserve-ci-e2e-test resourceVersion: "27610" uid: 45fb4487-0075-4197-90c4-7e33962b3ef6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:41:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: d6089f19-7a21-4543-9e4c-5ea91e9a5bb1 kind: Event lastTimestamp: "2026-07-28T10:42:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:55Z" name: error-404-isvc-f7711-predictor.18c66e0fad57700c namespace: kserve-ci-e2e-test resourceVersion: "27611" uid: ed815028-068e-458c-8e59-966b84eed6c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: d6089f19-7a21-4543-9e4c-5ea91e9a5bb1 kind: Event lastTimestamp: "2026-07-28T10:42:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:10Z" name: error-404-isvc-f7711-predictor.18c66e132c08610c namespace: kserve-ci-e2e-test resourceVersion: "27148" uid: 6ec80e7e-e8a0-49c2-b9fe-93ad768bf3ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: d6089f19-7a21-4543-9e4c-5ea91e9a5bb1 kind: Event lastTimestamp: "2026-07-28T10:42:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:10Z" name: error-404-isvc-f7711-predictor.18c66e132c518a22 namespace: kserve-ci-e2e-test resourceVersion: "27150" uid: 57528c1c-af2d-44b0-9e70-1b6164736b01 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: c9f0e8f3-8a77-4ab7-a16f-dbb1824391a1 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:30Z" name: error-404-isvc-f7711.18c66e0c2e873ce9 namespace: kserve-ci-e2e-test resourceVersion: "27371" uid: 552e7ad4-038c-476a-a672-8e81647e34c4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: c9f0e8f3-8a77-4ab7-a16f-dbb1824391a1 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7711": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7711": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-f7711.18c66e0c2f463237 namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: 0196daa4-0743-4f0d-8fc6-22ce50b999a9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: c9f0e8f3-8a77-4ab7-a16f-dbb1824391a1 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7711": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:40Z" name: error-404-isvc-f7711.18c66e0c2f467b67 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 2303ab40-b1e3-4632-88ac-fa1fb715f30f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27367" uid: c9f0e8f3-8a77-4ab7-a16f-dbb1824391a1 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: InferenceService [error-404-isvc-f7711] is Ready metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:30Z" name: error-404-isvc-f7711.18c66e17c4d56f2a namespace: kserve-ci-e2e-test resourceVersion: "27369" uid: c5624f54-7205-48af-87ec-69b6d01cd67d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:19:20.288478Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 to ip-10-0-130-25.ec2.internal metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd40bf57c95 namespace: kserve-ci-e2e-test resourceVersion: "14786" uid: acfd7a6c-a203-4623-b0cc-05f2040dc75f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd4252a1793 namespace: kserve-ci-e2e-test resourceVersion: "14855" uid: d71a1c0c-1bec-4990-9259-c37475da582c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:527f9c4ef06f4e1d255c6c4b87333be01c67680fe0b3417b8fb7aeb5d5b61bbf" already present on machine metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd427b99e37 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 769b28c6-1e5e-48dc-83fb-c7fa450584b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd42a733cec namespace: kserve-ci-e2e-test resourceVersion: "14871" uid: 7b99118c-ad7e-43f7-a09b-bdc280d453d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd42ad06fca namespace: kserve-ci-e2e-test resourceVersion: "14872" uid: a0ca2465-a26a-423b-87ad-71cf382aaa27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:25Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1813" metadata: creationTimestamp: "2026-07-28T10:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:25Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cd534da6c7a namespace: kserve-ci-e2e-test resourceVersion: "14994" uid: 61f32136-5bb8-4e66-811d-98f14cbadae6 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1813" in 23.568s (23.568s including waiting). Image size: 1555733417 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:48Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdab1a23c8d namespace: kserve-ci-e2e-test resourceVersion: "15226" uid: 03b07328-4ea1-4fe2-b1d8-7bbda9358fb6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:48Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdab65ff62d namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: d2074b2a-6354-4ad1-9f33-eb0fff1fb521 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:48Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdab77a583d namespace: kserve-ci-e2e-test resourceVersion: "15229" uid: 0663f8d5-43b5-40c3-bc02-39556bdfb98e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:48Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T10:19:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:48Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdab785d1f3 namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 87129df4-c990-4af0-9973-06a21c07c09c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.158s (3.158s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdb73cf5c36 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: 15dc3830-73de-4d54-86a0-93cf718f837a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdb795f4d0a namespace: kserve-ci-e2e-test resourceVersion: "15271" uid: 7de0fb90-077e-4aa4-aeaf-52c81033744e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdb7a4108d5 namespace: kserve-ci-e2e-test resourceVersion: "15272" uid: 36850259-6f41-4658-8eea-80295d4f1ed7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:20:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:58Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cdb870e85ce namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 4e7d4a75-1443-47b1-ad63-735b936ea9cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cf4aec9ca1e namespace: kserve-ci-e2e-test resourceVersion: "16513" uid: 7d3c3f63-67e2-4c00-b792-15af17390a31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cf4aeca443f namespace: kserve-ci-e2e-test resourceVersion: "16518" uid: d295c2bd-6fbe-4a2d-a28d-0a936dddd7ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 32af236a-7cfe-469a-9a45-e04f23144582 kind: Event lastTimestamp: "2026-07-28T10:21:43Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:21:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:43Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6.18c66cf55f6c59b8 namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: 029ac020-a85f-4a0f-9e90-7ba9bb2d5cd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-85cb8c5574 namespace: kserve-ci-e2e-test resourceVersion: "14771" uid: ef64f14a-8084-4dd8-8f14-20f5a69fe717 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-85cb8c5574-2ccq6' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor-85cb8c5574.18c66cd40b81a90f namespace: kserve-ci-e2e-test resourceVersion: "14781" uid: 75c57334-c040-411e-8058-18515c3576ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14770" uid: 0db07b2b-fa2c-4915-b5cb-d5d6801650b1 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-85cb8c5574 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-1-predictor.18c66cd4055ac8b2 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 9f8777f9-cb89-451e-920d-eb2aae755458 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 40810280-a460-4dc3-9e77-a475f794f611 kind: Event lastTimestamp: "2026-07-28T10:20:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:05Z" name: isvc-sklearn-graph-1-predictor.18c66cd78b69e915 namespace: kserve-ci-e2e-test resourceVersion: "15377" uid: 3855dbaf-57fc-4063-8a9e-8915dfcf3a88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14778" uid: 40810280-a460-4dc3-9e77-a475f794f611 kind: Event lastTimestamp: "2026-07-28T10:20:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:05Z" name: isvc-sklearn-graph-1-predictor.18c66cd78c146552 namespace: kserve-ci-e2e-test resourceVersion: "15378" uid: 5c19738c-d4d9-468a-a38f-aaa38eded8d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 40810280-a460-4dc3-9e77-a475f794f611 kind: Event lastTimestamp: "2026-07-28T10:21:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:35Z" name: isvc-sklearn-graph-1-predictor.18c66ce208ab3635 namespace: kserve-ci-e2e-test resourceVersion: "16367" uid: 70f3d28b-f086-4845-933b-d91ed7854e84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: 40810280-a460-4dc3-9e77-a475f794f611 kind: Event lastTimestamp: "2026-07-28T10:21:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:35Z" name: isvc-sklearn-graph-1-predictor.18c66ce2091581f1 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: fb5dac53-d8fa-4d34-966e-8ba4382c4156 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14761" uid: 3bc65d79-9d34-420a-b9b4-c6859b50e114 kind: Event lastTimestamp: "2026-07-28T10:21:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:08Z" name: isvc-sklearn-graph-1.18c66cd40b853025 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 3c85e69e-44ec-438c-beef-d137c1b200de reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 3bc65d79-9d34-420a-b9b4-c6859b50e114 kind: Event lastTimestamp: "2026-07-28T10:21:08Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T10:21:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:08Z" name: isvc-sklearn-graph-1.18c66ced40a15e66 namespace: kserve-ci-e2e-test resourceVersion: "16122" uid: 8302cf17-873f-4487-96a7-a6baea494ad9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:19:20.978124Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14899" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd43510aca0 namespace: kserve-ci-e2e-test resourceVersion: "14903" uid: 8edce60d-4f81-4adf-af6a-bb77606c8b95 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd4442dd68d namespace: kserve-ci-e2e-test resourceVersion: "14922" uid: fbd06996-b485-4d4f-b047-5e6f7e0003b3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:22Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:22Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:19:22Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd4731c6516 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 9f855a48-5d1e-41e1-b5ed-889ee2e145ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:22Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:22Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:527f9c4ef06f4e1d255c6c4b87333be01c67680fe0b3417b8fb7aeb5d5b61bbf" metadata: creationTimestamp: "2026-07-28T10:19:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:22Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd4753e7f51 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: d4e3a7db-b9a0-42d6-96eb-9bcd92755b12 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:25Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:527f9c4ef06f4e1d255c6c4b87333be01c67680fe0b3417b8fb7aeb5d5b61bbf" in 3.405s (3.405s including waiting). Image size: 302621120 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:25Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd54037d1f0 namespace: kserve-ci-e2e-test resourceVersion: "14999" uid: 5c743ffc-9245-4a3c-9870-54df1e65af7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T10:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:25Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd54385aa80 namespace: kserve-ci-e2e-test resourceVersion: "15001" uid: bb88b875-4fbe-43ba-9824-5abbcd86dd80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T10:19:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:25Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd54404300a namespace: kserve-ci-e2e-test resourceVersion: "15002" uid: 0fa1cc50-fd69-4641-8968-5a18f1ebbe08 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1813" metadata: creationTimestamp: "2026-07-28T10:19:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:30Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd6511c2730 namespace: kserve-ci-e2e-test resourceVersion: "15040" uid: 629473e5-2844-4955-8c13-08019412c353 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:42Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1813" in 12.756s (12.756s including waiting). Image size: 1555733417 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd9497b3011 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 24266a6e-1e46-474e-b88d-7fb44a3a014d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd94d4ee162 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: 5a3d5fb0-8221-4a44-8ac5-221858d14752 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd94dca1113 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: 370586a2-20d1-45f0-87bb-ee55b66ba2a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T10:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd94ddc4fe4 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: b4937387-39f8-4cf4-8007-b27f6d96d7d3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:45Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.188s (2.188s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:45Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd9d0562c2e namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 67006df6-cc8c-4310-9cd7-bdc4038086e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:45Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd9d396497f namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: da5aa59f-5ffc-43a1-84db-947760498baa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:19:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:19:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:45Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cd9d40fd265 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: b41a0164-5575-4ab5-828b-2125007c322e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T10:19:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:19:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cda10e5fafb namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 5e5c9c4a-e727-4010-96b1-2b9318eac9c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cf4ab0c7f55 namespace: kserve-ci-e2e-test resourceVersion: "16484" uid: 4b7a23cd-fe01-4c7c-a2f5-d0fe5f496adc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cf4ab1e8302 namespace: kserve-ci-e2e-test resourceVersion: "16487" uid: 2dfa4cc0-ffb1-4d71-a12a-bdf4799492eb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb namespace: kserve-ci-e2e-test resourceVersion: "14901" uid: 9cf6bd77-fe36-46d8-a45e-88ffc15f53ff kind: Event lastTimestamp: "2026-07-28T10:21:42Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:42Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb.18c66cf513417b8a namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: aa1da011-3e84-4f7c-a280-6572236a2c48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5bcddb87c8 namespace: kserve-ci-e2e-test resourceVersion: "14891" uid: 4238b70e-d0a7-4e51-9773-0cafd7775ba5 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5bcddb87c8-46rgb' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-2-predictor-5bcddb87c8.18c66cd4348176d0 namespace: kserve-ci-e2e-test resourceVersion: "14900" uid: 71f33410-23f4-4fdf-a533-79337fd4b81f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14889" uid: 6a927eab-52b6-4a64-b76c-8ffb1705d279 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5bcddb87c8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-sklearn-graph-2-predictor.18c66cd4319b6726 namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: 12507e8d-d828-43a5-9c02-a45590ebd1ae reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 3400dad9-c750-4c6f-8b45-65cb0e61dbbe kind: Event lastTimestamp: "2026-07-28T10:19:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:50Z" name: isvc-sklearn-graph-2-predictor.18c66cd7b0895dd6 namespace: kserve-ci-e2e-test resourceVersion: "15259" uid: 72ee06b1-5b77-4d35-a165-f0fc9eb72942 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14890" uid: 3400dad9-c750-4c6f-8b45-65cb0e61dbbe kind: Event lastTimestamp: "2026-07-28T10:19:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:50Z" name: isvc-sklearn-graph-2-predictor.18c66cd7b148b371 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 45b7ac0b-4711-4399-b871-bb6345b21d6b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:20:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 3400dad9-c750-4c6f-8b45-65cb0e61dbbe kind: Event lastTimestamp: "2026-07-28T10:21:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:36Z" name: isvc-sklearn-graph-2-predictor.18c66cdeaebb8f97 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 0c113370-0608-427d-9796-b6b3a0bceb0d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:20:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 3400dad9-c750-4c6f-8b45-65cb0e61dbbe kind: Event lastTimestamp: "2026-07-28T10:21:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:36Z" name: isvc-sklearn-graph-2-predictor.18c66cdeaf2875e7 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: 405e0527-0294-4d3c-bbc7-8fd6474e1759 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14873" uid: 39faacff-efff-4f4b-b787-e64e30ca13f6 kind: Event lastTimestamp: "2026-07-28T10:21:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:02Z" name: isvc-sklearn-graph-2.18c66cd431acf00c namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: d26c4d9c-4368-4928-b9b5-98a3137cc395 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16051" uid: 39faacff-efff-4f4b-b787-e64e30ca13f6 kind: Event lastTimestamp: "2026-07-28T10:21:02Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T10:21:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:02Z" name: isvc-sklearn-graph-2.18c66cebc86c0070 namespace: kserve-ci-e2e-test resourceVersion: "16052" uid: b26b255b-3383-4898-b52d-b7b94e9d22bb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:19:20.678604Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-srpkx to ip-10-0-130-25.ec2.internal metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:19:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd423365ec4 namespace: kserve-ci-e2e-test resourceVersion: "14847" uid: 0101622c-ca82-4048-83cd-d7b2d75c9a3a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd4306739d0 namespace: kserve-ci-e2e-test resourceVersion: "14888" uid: c3de3339-fabc-4ae9-89a2-49a957ee4ca6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:19:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd4604c267f namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 53a594db-9acc-46da-b6d3-20bd4a6e33e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:527f9c4ef06f4e1d255c6c4b87333be01c67680fe0b3417b8fb7aeb5d5b61bbf" already present on machine metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd461cc8a20 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: b9604bc8-ca84-4967-bfb6-63e6041b2105 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd4649ce3f9 namespace: kserve-ci-e2e-test resourceVersion: "14948" uid: 5c0e63a7-9711-4cef-a242-ef696c11b556 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd464faa288 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 7389d0f4-35da-4e9b-be5c-a312088218f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:26Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T10:19:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cd57148285c namespace: kserve-ci-e2e-test resourceVersion: "15012" uid: 59db19e3-43ad-47f0-bb2a-93108bbe0b45 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:49Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 22.962s (22.962s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdac9f71218 namespace: kserve-ci-e2e-test resourceVersion: "15233" uid: 5057c8e4-c4f3-420d-a974-f4bfeccac341 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdace148532 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: cd8fff6c-fb90-4291-ba9c-49e1c06bd142 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdacebfaf0e namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 3a24b533-e0a5-46cd-94d3-34954c7ae730 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:49Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T10:19:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:49Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdacecb5269 namespace: kserve-ci-e2e-test resourceVersion: "15236" uid: a316951a-c3e3-4567-8d40-21ec9579d42b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.776s (2.776s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdb7450d905 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: a3ee9c26-73f9-446e-b3ef-e25c0e02ebcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdb7a4ab979 namespace: kserve-ci-e2e-test resourceVersion: "15273" uid: afa3990f-cdb9-4d7c-941b-f7a0f1d263cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:19:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:52Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdb7ae1f0bb namespace: kserve-ci-e2e-test resourceVersion: "15274" uid: b85ef896-e31b-4947-b461-68a02594753a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:19:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:20:38Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:19:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:38Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cdb86f49fed namespace: kserve-ci-e2e-test resourceVersion: "15662" uid: 3540011a-73fb-4de4-9179-def5fc8fb013 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cf4a852cea8 namespace: kserve-ci-e2e-test resourceVersion: "16462" uid: 5fd01925-30cc-48cb-b669-a4c2acad2b94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cf4a8534294 namespace: kserve-ci-e2e-test resourceVersion: "16464" uid: bac803e9-33fc-4104-839e-ee357afb04c5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-srpkx namespace: kserve-ci-e2e-test resourceVersion: "14844" uid: 52f4e69e-7acb-4a5e-bcc0-c13c9f3ffcd4 kind: Event lastTimestamp: "2026-07-28T10:21:43Z" message: 'Readiness probe failed: Get "https://10.133.0.23:8643/healthz": dial tcp 10.133.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:21:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-srpkx.18c66cf55f6d1e88 namespace: kserve-ci-e2e-test resourceVersion: "16611" uid: d4a68168-faba-4f13-9436-ec43afbc34f5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-130-25.ec2.internal source: component: kubelet host: ip-10-0-130-25.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14834" uid: ca981ee6-3027-4d83-99fd-806ca82c3e80 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-srpkx' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c66cd4225e3680 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 4538518c-2acc-4620-b130-834ee1057d11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: b16560ec-459f-4731-ae49-8344178bbb5c kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: isvc-xgboost-graph-predictor.18c66cd41d46528e namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: 0af9c2ca-ae24-4bb1-9cd1-108b6a96cbc8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: f8248acb-a6d6-4254-b8ae-f36e84305a93 kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: isvc-xgboost-graph-predictor.18c66cd79a3b3d7a namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: 136f7491-4108-40f7-8706-ba29bfc3baea reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: f8248acb-a6d6-4254-b8ae-f36e84305a93 kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: isvc-xgboost-graph-predictor.18c66cd79ab4cb9a namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: 2e6beffe-c137-464f-a42b-caf75767f0b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:20:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f8248acb-a6d6-4254-b8ae-f36e84305a93 kind: Event lastTimestamp: "2026-07-28T10:21:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:20Z" name: isvc-xgboost-graph-predictor.18c66ce216f15a05 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: a4e533ee-71ab-4521-815d-0e49f486097f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:20:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: f8248acb-a6d6-4254-b8ae-f36e84305a93 kind: Event lastTimestamp: "2026-07-28T10:21:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:20Z" name: isvc-xgboost-graph-predictor.18c66ce217570c60 namespace: kserve-ci-e2e-test resourceVersion: "16219" uid: b33f60bf-ed26-4149-8f4c-1fad18ac1631 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 1013e35f-7825-432b-b924-8742aed6367f kind: Event lastTimestamp: "2026-07-28T10:20:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:48Z" name: isvc-xgboost-graph.18c66cd41c1dcce6 namespace: kserve-ci-e2e-test resourceVersion: "15782" uid: 3a2af435-3e3e-4bf4-9fea-f5e798bb168b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 1013e35f-7825-432b-b924-8742aed6367f kind: Event lastTimestamp: "2026-07-28T10:20:48Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T10:20:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:48Z" name: isvc-xgboost-graph.18c66ce896e9888e namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 59ce921b-b46d-42f7-a03d-0287db840c6c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:21:30.166881Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-5595dc65c9-w4vhv to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:21:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:21:30Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf24950abb0 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: 98d194d5-2d4f-4d57-adeb-031fbcde7cb9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:30Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:30Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf2553e03b0 namespace: kserve-ci-e2e-test resourceVersion: "16316" uid: a8d7fc32-e662-4410-81e9-c7eb60cd443b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:31Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:21:31Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf2870ad720 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: d81dceb6-2dd6-44a2-810d-32e502b12de3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:31Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf28845bc00 namespace: kserve-ci-e2e-test resourceVersion: "16327" uid: a49c4413-222e-443b-8cd9-ce27aa7a1da0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:31Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T10:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:31Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf28b7dfe15 namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 11a7ba1b-12b8-4a7e-a7eb-c27c9c3ebbea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:31Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T10:21:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:31Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf28befb835 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: 806e92d2-4d08-4064-b590-869df2151eea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf4a2e14b1c namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: 2ce87338-f0f1-4295-b7ab-db97dc64ab5c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:21:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-5595dc65c9-w4vhv namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 6e8e8838-a79f-43e0-b034-beaba3d794e3 kind: Event lastTimestamp: "2026-07-28T10:22:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:21:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:07Z" name: model-chainer-5595dc65c9-w4vhv.18c66cf530df9020 namespace: kserve-ci-e2e-test resourceVersion: "16869" uid: 24da67e4-b87d-40cc-92d8-543021ad57cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-5595dc65c9 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: e8080b9a-37e5-4f8e-a1e8-e96d377023ba kind: Event lastTimestamp: "2026-07-28T10:21:30Z" message: 'Created pod: model-chainer-5595dc65c9-w4vhv' metadata: creationTimestamp: "2026-07-28T10:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:30Z" name: model-chainer-5595dc65c9.18c66cf24895f29b namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: d018cd0a-de31-456a-a886-297bbb84d429 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: 199c0332-594c-46dd-ad8a-72255a96f8ca kind: Event lastTimestamp: "2026-07-28T10:21:30Z" message: Scaled up replica set model-chainer-5595dc65c9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:30Z" name: model-chainer.18c66cf2479ecd70 namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 376229af-db92-43c8-97c8-cd50fcb39b78 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 51ee1f10-8bf1-416d-8cfc-904fd600ea57 kind: Event lastTimestamp: "2026-07-28T10:21:30Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T10:21:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:30Z" name: model-chainer.18c66cf24b5e1415 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 25f80fbe-5a31-4d92-83ca-3596344b2d85 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:22:40.466012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17139" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0b9d4-7db5cc7799-hgsps to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:22:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:22:40Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02a7783ddb namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: 9524ff16-b02f-450e-9ca9-2f64f521c355 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:40Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-0b9d4-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:40Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02b3ba7c88 namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 7da4d7b6-dc6a-4417-b2a0-f8a826914f78 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:41Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17149" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:22:41Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:22:41Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02e5d96bc8 namespace: kserve-ci-e2e-test resourceVersion: "17170" uid: 705cf119-838e-4553-b5c9-d24494c25303 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0b9d4} kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:22:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:41Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02e73ee2d2 namespace: kserve-ci-e2e-test resourceVersion: "17172" uid: fea94a51-7f44-43fb-ba10-2d2dbcbc4e7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0b9d4} kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:22:41Z" message: 'Created container: sequence-graph-0b9d4' metadata: creationTimestamp: "2026-07-28T10:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:41Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02ea540760 namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 7bca6127-8806-4ae7-b64a-2c8c4aa1a9fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0b9d4} kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:22:41Z" message: Started container sequence-graph-0b9d4 metadata: creationTimestamp: "2026-07-28T10:22:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:22:41Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d02eac9c0ee namespace: kserve-ci-e2e-test resourceVersion: "17174" uid: a2728ebe-29d9-47f0-8589-8754c27d9540 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0b9d4} kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Stopping container sequence-graph-0b9d4 metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d75cc38189b namespace: kserve-ci-e2e-test resourceVersion: "20808" uid: 6fb3ae9f-f9b2-4d5b-809e-e639022192c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0b9d4} kind: Pod name: sequence-graph-0b9d4-7db5cc7799-hgsps namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: 6c7d158e-7c07-49af-97b1-121a8e688748 kind: Event lastTimestamp: "2026-07-28T10:31:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:22Z" name: sequence-graph-0b9d4-7db5cc7799-hgsps.18c66d767d3bdf87 namespace: kserve-ci-e2e-test resourceVersion: "21347" uid: 845650e8-cb15-40f2-ac66-6312a085fb83 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0b9d4-7db5cc7799 namespace: kserve-ci-e2e-test resourceVersion: "17135" uid: d79199a5-17c0-42ef-b51d-2379ae80ecb9 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'Created pod: sequence-graph-0b9d4-7db5cc7799-hgsps' metadata: creationTimestamp: "2026-07-28T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: sequence-graph-0b9d4-7db5cc7799.18c66d02a703e307 namespace: kserve-ci-e2e-test resourceVersion: "17140" uid: 11f5bfb2-7e15-45b3-8b3b-d9e880b71335 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17133" uid: 8482941b-4982-4da6-a0dd-e721f882b028 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: Scaled up replica set sequence-graph-0b9d4-7db5cc7799 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: sequence-graph-0b9d4.18c66d02a61a52c1 namespace: kserve-ci-e2e-test resourceVersion: "17136" uid: b425f356-e397-4741-8af3-318e12ace956 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17152" uid: 6f676e7a-29a6-480c-b5f5-efaa36db008c kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: InferenceGraph [sequence-graph-0b9d4] is Ready metadata: creationTimestamp: "2026-07-28T10:22:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:22:40Z" name: sequence-graph-0b9d4.18c66d02aafa8ffd namespace: kserve-ci-e2e-test resourceVersion: "17154" uid: d1f80e28-ae07-4cc1-9f95-f9e2226b68df reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 06d4d040-f8e3-4e5e-b663-cabcfd183788 kind: Event lastTimestamp: "2026-07-28T10:23:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:23:10Z" name: sequence-graph-0b9d4.18c66d06254bab12 namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: e50b5da0-90f0-4a23-90bc-f1901ea63666 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:22:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17134" uid: 06d4d040-f8e3-4e5e-b663-cabcfd183788 kind: Event lastTimestamp: "2026-07-28T10:23:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:23:10Z" name: sequence-graph-0b9d4.18c66d0625bb171b namespace: kserve-ci-e2e-test resourceVersion: "17382" uid: 6fd65e7a-717a-452d-8a8a-8a98fa12206f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:32:05.208875Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21736" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-222a3-5f5974b4b4-bb4dl to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:32:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:32:05Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d8624c47ee8 namespace: kserve-ci-e2e-test resourceVersion: "21741" uid: 665a3ca9-ab82-45b4-9999-e28b98447893 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:05Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:05Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-222a3-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:05Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d862ec7d4d1 namespace: kserve-ci-e2e-test resourceVersion: "21753" uid: 74d6c56a-045c-419f-8315-a4f2204897d8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21746" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:06Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:32:06Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d8662410a4c namespace: kserve-ci-e2e-test resourceVersion: "21762" uid: efc0261b-2d10-46db-8e31-6ccda9f8862f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-222a3} kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:06Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d86639470ef namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 3900d1b7-446c-4142-bc5d-ef458f035fff reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-222a3} kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:06Z" message: 'Created container: sequence-graph-222a3' metadata: creationTimestamp: "2026-07-28T10:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:06Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d866726d427 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 75042ede-ec31-4a74-8499-b7dd0604386c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-222a3} kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:06Z" message: Started container sequence-graph-222a3 metadata: creationTimestamp: "2026-07-28T10:32:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:06Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d86679c591e namespace: kserve-ci-e2e-test resourceVersion: "21767" uid: 40b2f2c4-bf26-417d-b0c9-863502241cc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-222a3} kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Stopping container sequence-graph-222a3 metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d88806fad57 namespace: kserve-ci-e2e-test resourceVersion: "21853" uid: fe60810a-3c9f-48bd-8533-897dcb6b13a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-222a3} kind: Pod name: sequence-graph-222a3-5f5974b4b4-bb4dl namespace: kserve-ci-e2e-test resourceVersion: "21739" uid: 4d475df7-d3a4-4237-8b78-08843f4ef33b kind: Event lastTimestamp: "2026-07-28T10:32:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:42Z" name: sequence-graph-222a3-5f5974b4b4-bb4dl.18c66d890181ed90 namespace: kserve-ci-e2e-test resourceVersion: "22237" uid: 3dd2594a-f3b2-4e21-a550-0a1fe39d270d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-222a3-5f5974b4b4 namespace: kserve-ci-e2e-test resourceVersion: "21733" uid: 6bdd1976-36d3-41bc-b633-fd60acf66555 kind: Event lastTimestamp: "2026-07-28T10:32:05Z" message: 'Created pod: sequence-graph-222a3-5f5974b4b4-bb4dl' metadata: creationTimestamp: "2026-07-28T10:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:05Z" name: sequence-graph-222a3-5f5974b4b4.18c66d86244d421e namespace: kserve-ci-e2e-test resourceVersion: "21738" uid: 0d6547ff-8064-405f-8289-e1a394df3e86 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-222a3 namespace: kserve-ci-e2e-test resourceVersion: "21731" uid: 88f11e5a-679e-4f27-979e-d10df64a96ca kind: Event lastTimestamp: "2026-07-28T10:32:05Z" message: Scaled up replica set sequence-graph-222a3-5f5974b4b4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:05Z" name: sequence-graph-222a3.18c66d8623645157 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: c32a909d-cc62-4e82-8a1e-c55a0fb19b16 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-222a3 namespace: kserve-ci-e2e-test resourceVersion: "21749" uid: be5cf117-49e8-4821-8dc6-01f583dae090 kind: Event lastTimestamp: "2026-07-28T10:32:05Z" message: InferenceGraph [sequence-graph-222a3] is Ready metadata: creationTimestamp: "2026-07-28T10:32:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:05Z" name: sequence-graph-222a3.18c66d8628d24f6b namespace: kserve-ci-e2e-test resourceVersion: "21751" uid: fd0521a1-36a0-478f-8de7-3db46b3f80c2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:33:15.559140Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22518" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8d409-85cbb7d94-dp7l8 to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:33:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:33:15Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d9685f85b7f namespace: kserve-ci-e2e-test resourceVersion: "22523" uid: 4d8c1635-c10b-4111-b639-fad659cc22bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:33:15Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:33:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-8d409-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:33:16Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d9693e2f5da namespace: kserve-ci-e2e-test resourceVersion: "22546" uid: 915eac41-615e-43f5-9eb8-63f88384b3fd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22527" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:33:17Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:33:17Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d96f9371b0d namespace: kserve-ci-e2e-test resourceVersion: "22557" uid: 17e03adc-1a0d-4136-8c5b-1d9becbfc52e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d409} kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:33:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:33:17Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d96fa93a04e namespace: kserve-ci-e2e-test resourceVersion: "22560" uid: f5f038cc-1043-4217-bbd1-4ff7f7f0ca04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d409} kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:33:17Z" message: 'Created container: sequence-graph-8d409' metadata: creationTimestamp: "2026-07-28T10:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:33:17Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d96fd6c7b44 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: 04a8f51f-d7d9-4ac5-b559-7b8be7d3188b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d409} kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:33:17Z" message: Started container sequence-graph-8d409 metadata: creationTimestamp: "2026-07-28T10:33:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:33:17Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66d96fe296725 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 02792738-ecc2-4e58-9aec-e873eb65ef62 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d409} kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Stopping container sequence-graph-8d409 metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66e09a6cb305e namespace: kserve-ci-e2e-test resourceVersion: "26499" uid: be1197d3-32ae-4aa1-ab85-74ef83b649ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8d409} kind: Pod name: sequence-graph-8d409-85cbb7d94-dp7l8 namespace: kserve-ci-e2e-test resourceVersion: "22521" uid: 4cdd871e-21d2-4659-ac46-94c67c2d8b66 kind: Event lastTimestamp: "2026-07-28T10:41:58Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:58Z" name: sequence-graph-8d409-85cbb7d94-dp7l8.18c66e0a88f2a6e6 namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: 18fc1466-57e8-4e35-8a04-5f9c7079eaec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8d409-85cbb7d94 namespace: kserve-ci-e2e-test resourceVersion: "22514" uid: 0db28c9e-d174-43e1-9a00-077ab41f62d8 kind: Event lastTimestamp: "2026-07-28T10:33:15Z" message: 'Created pod: sequence-graph-8d409-85cbb7d94-dp7l8' metadata: creationTimestamp: "2026-07-28T10:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:15Z" name: sequence-graph-8d409-85cbb7d94.18c66d9685607e05 namespace: kserve-ci-e2e-test resourceVersion: "22520" uid: ef3b774c-96c0-41d2-991b-1c2f7fa13bb6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22511" uid: 4c9f37df-b137-40cd-a347-7958abcf7acd kind: Event lastTimestamp: "2026-07-28T10:33:15Z" message: Scaled up replica set sequence-graph-8d409-85cbb7d94 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:15Z" name: sequence-graph-8d409.18c66d96844baeac namespace: kserve-ci-e2e-test resourceVersion: "22516" uid: c2394159-d3c4-4e69-9dbf-fea14b7c2f8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:15Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22529" uid: 3b85db71-8fb8-4f5a-9aa0-e75c15fcbf87 kind: Event lastTimestamp: "2026-07-28T10:33:15Z" message: InferenceGraph [sequence-graph-8d409] is Ready metadata: creationTimestamp: "2026-07-28T10:33:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:33:15Z" name: sequence-graph-8d409.18c66d9687ee19aa namespace: kserve-ci-e2e-test resourceVersion: "22531" uid: ab50eb8b-89c1-4d1b-b749-9f8db81a3b9c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:33:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 702136ae-aa84-4f2d-ad2b-4a9459a4c9c0 kind: Event lastTimestamp: "2026-07-28T10:33:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:45Z" name: sequence-graph-8d409.18c66d9a02fccc1e namespace: kserve-ci-e2e-test resourceVersion: "22770" uid: a3eb727c-70b9-4bf9-9629-63386bacb98a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:33:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22513" uid: 702136ae-aa84-4f2d-ad2b-4a9459a4c9c0 kind: Event lastTimestamp: "2026-07-28T10:33:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:33:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:45Z" name: sequence-graph-8d409.18c66d9a0365c9bc namespace: kserve-ci-e2e-test resourceVersion: "22771" uid: e2b260e7-6382-49c0-b68e-db3a1cf8fb1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:41:30.586914Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-e803a-55fbf78854-lnmxf to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:41:30Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e09c7ecb335 namespace: kserve-ci-e2e-test resourceVersion: "26611" uid: 98a32c33-f00d-43e5-a8f4-fb2006557219 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26616" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:41:31Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e09e1cd41b6 namespace: kserve-ci-e2e-test resourceVersion: "26639" uid: 5aa24186-36b6-4c78-8d7e-d6c2f4308800 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e803a} kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e09e310f905 namespace: kserve-ci-e2e-test resourceVersion: "26641" uid: 3ea2b758-58af-4ece-ad7c-f3fb9f2932a7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e803a} kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: 'Created container: splitter-graph-e803a' metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e09e6c4fbc4 namespace: kserve-ci-e2e-test resourceVersion: "26644" uid: fa78f905-31bf-4e60-91ec-4f0b340214ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e803a} kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:41:31Z" message: Started container splitter-graph-e803a metadata: creationTimestamp: "2026-07-28T10:41:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:31Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e09e73bfe36 namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: 477fde7b-34ed-443a-9644-f353f2cb11b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e803a} kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Stopping container splitter-graph-e803a metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e0c20710370 namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: c8f6e1bd-10d2-4cd2-ab66-153bee7cda56 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-e803a} kind: Pod name: splitter-graph-e803a-55fbf78854-lnmxf namespace: kserve-ci-e2e-test resourceVersion: "26607" uid: e5fabe5e-9258-4f06-be3a-2c511f06620c kind: Event lastTimestamp: "2026-07-28T10:42:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:08Z" name: splitter-graph-e803a-55fbf78854-lnmxf.18c66e0cad73d9d3 namespace: kserve-ci-e2e-test resourceVersion: "27121" uid: bdd99558-a882-4920-9265-e46079c8dde1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-e803a-55fbf78854 namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: 8f10abac-f6cf-40d8-bc9c-574193536a1d kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Created pod: splitter-graph-e803a-55fbf78854-lnmxf' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: splitter-graph-e803a-55fbf78854.18c66e09c77c3e31 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: 7bfa27ae-3148-4dd0-9d04-337feb9b7132 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-e803a namespace: kserve-ci-e2e-test resourceVersion: "26600" uid: 47fbd26e-c420-47e1-9de9-983914aa8097 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Scaled up replica set splitter-graph-e803a-55fbf78854 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: splitter-graph-e803a.18c66e09c6c22fb8 namespace: kserve-ci-e2e-test resourceVersion: "26603" uid: f2b9af75-dc95-4e26-8253-62d7be807545 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-e803a namespace: kserve-ci-e2e-test resourceVersion: "26618" uid: 7f751276-1ba2-4a8c-932b-8a832e57f347 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: InferenceGraph [splitter-graph-e803a] is Ready metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:30Z" name: splitter-graph-e803a.18c66e09cab7ab75 namespace: kserve-ci-e2e-test resourceVersion: "26620" uid: 9e03d371-7dbb-42bb-8576-7f52bb22e48c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:42:40.883706Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27464" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-f7711-5d9485b744-t55xb to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:42:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:42:40Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e1a25f0a659 namespace: kserve-ci-e2e-test resourceVersion: "27469" uid: 5c9ab662-520c-4a4f-8977-4ca2fe2e5e8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27472" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:42:41Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:42:41Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e1a3f8fab3d namespace: kserve-ci-e2e-test resourceVersion: "27485" uid: 743dbd20-ae98-465a-b077-e9b34f8bf187 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7711} kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:42:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:41Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e1a40cb794b namespace: kserve-ci-e2e-test resourceVersion: "27487" uid: 2c97f549-0488-4934-bb70-0ac98e10b11f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7711} kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:42:41Z" message: 'Created container: splitter-graph-f7711' metadata: creationTimestamp: "2026-07-28T10:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:41Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e1a43df315b namespace: kserve-ci-e2e-test resourceVersion: "27489" uid: 1a5cf788-935d-4ada-916c-7221e30e3d67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7711} kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:42:41Z" message: Started container splitter-graph-f7711 metadata: creationTimestamp: "2026-07-28T10:42:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:41Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e1a44576fbf namespace: kserve-ci-e2e-test resourceVersion: "27490" uid: f6e14af7-0a28-4745-b432-3b806b13de92 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7711} kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:50:55Z" message: Stopping container splitter-graph-f7711 metadata: creationTimestamp: "2026-07-28T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:50:55Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e8d51d01a00 namespace: kserve-ci-e2e-test resourceVersion: "30974" uid: f479b2b0-547a-4a7a-9045-2b1ce3a30ca6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:50:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-f7711} kind: Pod name: splitter-graph-f7711-5d9485b744-t55xb namespace: kserve-ci-e2e-test resourceVersion: "27466" uid: 4052d564-bd42-4c3d-9d80-90162ce959ca kind: Event lastTimestamp: "2026-07-28T10:51:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:50:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:51:23Z" name: splitter-graph-f7711-5d9485b744-t55xb.18c66e8df8d2ab2f namespace: kserve-ci-e2e-test resourceVersion: "31210" uid: 98321061-41f4-457d-8b7a-279354ca9cbe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-f7711-5d9485b744 namespace: kserve-ci-e2e-test resourceVersion: "27460" uid: 881379b8-a4d4-49d3-ac5e-b85269d1d0d5 kind: Event lastTimestamp: "2026-07-28T10:42:40Z" message: 'Created pod: splitter-graph-f7711-5d9485b744-t55xb' metadata: creationTimestamp: "2026-07-28T10:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:40Z" name: splitter-graph-f7711-5d9485b744.18c66e1a257f4d95 namespace: kserve-ci-e2e-test resourceVersion: "27465" uid: 23fb9701-a3fc-4792-9023-f6047c685141 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27457" uid: 1f84c332-bdad-4c31-9f7c-c259e73223a8 kind: Event lastTimestamp: "2026-07-28T10:42:40Z" message: Scaled up replica set splitter-graph-f7711-5d9485b744 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:40Z" name: splitter-graph-f7711.18c66e1a24d502c4 namespace: kserve-ci-e2e-test resourceVersion: "27462" uid: 13f60f71-6c87-4811-be58-d0f6cfb51914 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27475" uid: 43916d93-2eda-473b-918e-a6b2bf61f848 kind: Event lastTimestamp: "2026-07-28T10:42:40Z" message: InferenceGraph [splitter-graph-f7711] is Ready metadata: creationTimestamp: "2026-07-28T10:42:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:40Z" name: splitter-graph-f7711.18c66e1a29f30258 namespace: kserve-ci-e2e-test resourceVersion: "27477" uid: a95bb09d-df56-4b5f-bb81-cfa1b591e44a reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:42:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 29643bdc-8b08-4437-a91b-256c67b03a10 kind: Event lastTimestamp: "2026-07-28T10:43:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:43:25Z" name: splitter-graph-f7711.18c66e1da35d4ce0 namespace: kserve-ci-e2e-test resourceVersion: "27823" uid: de33fc45-fa6b-49be-8625-d7882b48ffa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:42:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27458" uid: 29643bdc-8b08-4437-a91b-256c67b03a10 kind: Event lastTimestamp: "2026-07-28T10:43:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:43:25Z" name: splitter-graph-f7711.18c66e1da3a6cb00 namespace: kserve-ci-e2e-test resourceVersion: "27824" uid: 2d710df5-9adf-4dec-bca5-b31bd2e00b6e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:21:40.456317Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16506" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:21:40Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4ae9d051b namespace: kserve-ci-e2e-test resourceVersion: "16516" uid: 59f4f153-7137-4c9f-8a87-836d85450b50 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0b9d4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:40Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4bb2fada1 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: 4197535d-cc7a-4ed1-b983-4451f5125a9f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16522" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4ebb73cb9 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: ae0fe149-e2d5-4114-93d4-ea02a9d02645 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4ecf634fb namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: bec08ca1-e7a3-4bb5-9416-b6446b310b7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4efa8bfb0 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 69c4aa9a-4d9d-429a-b030-325c238a4a95 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4f0560029 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: b97d340e-7ce9-46e5-82f5-9c29a0531647 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4f05f34f1 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 09eed785-52d5-48f1-9775-ff8c520e5557 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4f2cc9737 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 0941c9b3-c951-408e-94be-919d364d44e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:21:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:41Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf4f32e7c18 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 6ce3e925-90a5-4f8b-824b-fdd5d720d394 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:21:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:30:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:57Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66cf5089738de namespace: kserve-ci-e2e-test resourceVersion: "20956" uid: cc5db55d-a581-4965-81b8-4507e29cbca0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66d75d2987a81 namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 18656a03-18e1-4568-947e-d4d3700ff049 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66d75d2993cd2 namespace: kserve-ci-e2e-test resourceVersion: "20831" uid: 1ee31164-4fe7-46f0-bf47-e785a5f8cb59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99 namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 21380bc3-7d5d-4d24-987d-6013ac575626 kind: Event lastTimestamp: "2026-07-28T10:30:57Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:57Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99.18c66d767d1b31f1 namespace: kserve-ci-e2e-test resourceVersion: "20954" uid: 91be9a71-93ef-4180-9463-04a4b35f3caa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0b9d4-predictor-54b67cb57d namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: a2f5255c-7fc3-4854-8280-e4f560c19ff7 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: 'Created pod: success-200-isvc-0b9d4-predictor-54b67cb57d-48n99' metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:40Z" name: success-200-isvc-0b9d4-predictor-54b67cb57d.18c66cf4ae0344e7 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 248727ae-7d7a-40cb-a766-a77f4b96d439 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 79a323f0-5b68-46f9-9863-0100f8b31ff8 kind: Event lastTimestamp: "2026-07-28T10:21:40Z" message: Scaled up replica set success-200-isvc-0b9d4-predictor-54b67cb57d from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:40Z" name: success-200-isvc-0b9d4-predictor.18c66cf4ac2af5c1 namespace: kserve-ci-e2e-test resourceVersion: "16490" uid: b9c7b78e-c910-4e0a-8500-fc726928d2b2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: c662220e-fc57-4a17-bfb8-368e8c1ffe16 kind: Event lastTimestamp: "2026-07-28T10:22:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:10Z" name: success-200-isvc-0b9d4-predictor.18c66cf82acaf2d2 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 9f525170-9f8b-4721-91d5-4108e75c0307 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:21:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: c662220e-fc57-4a17-bfb8-368e8c1ffe16 kind: Event lastTimestamp: "2026-07-28T10:22:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:10Z" name: success-200-isvc-0b9d4-predictor.18c66cf82b3a971b namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 4031295c-e1fc-4f01-8f0c-2624e8028fad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: c662220e-fc57-4a17-bfb8-368e8c1ffe16 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: success-200-isvc-0b9d4-predictor.18c66cff2905e666 namespace: kserve-ci-e2e-test resourceVersion: "17148" uid: ab915c87-41b1-4365-b3e0-51710b9f8bd5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:22:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0b9d4-predictor namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: c662220e-fc57-4a17-bfb8-368e8c1ffe16 kind: Event lastTimestamp: "2026-07-28T10:22:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:22:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:40Z" name: success-200-isvc-0b9d4-predictor.18c66cff29753586 namespace: kserve-ci-e2e-test resourceVersion: "17150" uid: 4146bd15-bb96-474c-a4ea-266de225b99b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:21:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 29d4b796-d7d6-433f-a6a7-6ed4c819fbc6 kind: Event lastTimestamp: "2026-07-28T10:22:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:21:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:22:28Z" name: success-200-isvc-0b9d4.18c66cf4ac6c652a namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: df450e40-96e7-4478-831b-7b0004d34528 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0b9d4 namespace: kserve-ci-e2e-test resourceVersion: "17038" uid: 29d4b796-d7d6-433f-a6a7-6ed4c819fbc6 kind: Event lastTimestamp: "2026-07-28T10:22:28Z" message: InferenceService [success-200-isvc-0b9d4] is Ready metadata: creationTimestamp: "2026-07-28T10:22:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:22:28Z" name: success-200-isvc-0b9d4.18c66cffc4255ee6 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: fe96c7b4-30e2-41c5-86b3-dc264d124c99 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:30:55.183364Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20857" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d75d6ec0840 namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 624fce39-8cd2-4d44-8061-9e6b863b27e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-222a3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d75df5f8c7d namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: 2fd43e39-a176-40d9-a43a-0d836eb98f97 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d7614463238 namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: e097a938-8c76-441d-b16c-e2b9178acf39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d7615b132c2 namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 015bc3f9-17ed-4933-a9f8-3d7351ffc813 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d7619400397 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 3ce26a3c-544a-43ea-90dd-853d0613187e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d761a4c70ef namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 6b028933-0208-4965-ad69-e6953d741db5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d761a609cf3 namespace: kserve-ci-e2e-test resourceVersion: "20923" uid: 614a3533-023f-46d5-8320-aaa3980acb3f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d761da4c3aa namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: c73deb33-4cc2-44d0-8c05-83aee27c7447 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:30:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:56Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d761e2828b4 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 631fe3d6-9b06-4edd-90f7-212ea77aa34a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:30:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:31:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:43Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d7647fcdf86 namespace: kserve-ci-e2e-test resourceVersion: "21519" uid: b057f666-701e-4405-9a67-9c4acaadf24d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d8888666244 namespace: kserve-ci-e2e-test resourceVersion: "21900" uid: f7e8b130-925f-4639-8049-97ae05494521 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d888866d52b namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: fd7e579a-9469-44fb-9ac1-46abd6b77e61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26 namespace: kserve-ci-e2e-test resourceVersion: "20861" uid: df58082d-bb54-4891-9fde-dd207abea5a4 kind: Event lastTimestamp: "2026-07-28T10:32:18Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:32:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:18Z" name: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26.18c66d8924313cf9 namespace: kserve-ci-e2e-test resourceVersion: "22039" uid: 197576f2-44ff-47bb-9168-41f6f8499122 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-222a3-predictor-6d5d87c54d namespace: kserve-ci-e2e-test resourceVersion: "20843" uid: 9a258fde-cb08-4099-b66c-e092404d80b4 kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'Created pod: success-200-isvc-222a3-predictor-6d5d87c54d-2tg26' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3-predictor-6d5d87c54d.18c66d75d66d5367 namespace: kserve-ci-e2e-test resourceVersion: "20860" uid: 28cab0b3-543d-4f1e-8f03-4bec4ad5ae4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 8bdb4344-59a9-49e0-b728-0a953841cc1f kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: Scaled up replica set success-200-isvc-222a3-predictor-6d5d87c54d from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3-predictor.18c66d75d4802f2e namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: d48bbbe4-17d1-480a-a13d-23e02d4a6cfd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:31:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 415714c3-9619-4e79-a179-fd73f9671258 kind: Event lastTimestamp: "2026-07-28T10:31:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:25Z" name: success-200-isvc-222a3-predictor.18c66d7952e0fa8c namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 1c6b214e-ba7e-4609-a2e4-ad63004a7b94 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:31:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 415714c3-9619-4e79-a179-fd73f9671258 kind: Event lastTimestamp: "2026-07-28T10:31:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:25Z" name: success-200-isvc-222a3-predictor.18c66d79535312ee namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 7fe7c4ba-bed8-43d6-906b-041ab13c36f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 415714c3-9619-4e79-a179-fd73f9671258 kind: Event lastTimestamp: "2026-07-28T10:32:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:10Z" name: success-200-isvc-222a3-predictor.18c66d80510b66ca namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: e697b859-36ce-4a13-9d13-9362dedd202f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:31:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-222a3-predictor namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 415714c3-9619-4e79-a179-fd73f9671258 kind: Event lastTimestamp: "2026-07-28T10:32:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:31:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:10Z" name: success-200-isvc-222a3-predictor.18c66d80517963fa namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 2c78f089-7c3a-464b-ade9-62791ed64c63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: ccfb794a-5e64-48e0-b200-1157c198c78b kind: Event lastTimestamp: "2026-07-28T10:31:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:53Z" name: success-200-isvc-222a3.18c66d75d47b201e namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: cb917d79-46b9-4fa8-8750-e988a5810221 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: ccfb794a-5e64-48e0-b200-1157c198c78b kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-222a3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-222a3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3.18c66d75d581c0a3 namespace: kserve-ci-e2e-test resourceVersion: "20855" uid: 00e5998d-bf70-4c60-9467-6623c95a95cf reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: ccfb794a-5e64-48e0-b200-1157c198c78b kind: Event lastTimestamp: "2026-07-28T10:30:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-222a3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:30:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:55Z" name: success-200-isvc-222a3.18c66d75d582117a namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: c3c2958a-94c3-447a-ae03-9ce04a0bf899 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-222a3 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: ccfb794a-5e64-48e0-b200-1157c198c78b kind: Event lastTimestamp: "2026-07-28T10:31:53Z" message: InferenceService [success-200-isvc-222a3] is Ready metadata: creationTimestamp: "2026-07-28T10:31:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:31:53Z" name: success-200-isvc-222a3.18c66d835766fe72 namespace: kserve-ci-e2e-test resourceVersion: "21610" uid: 34f665ac-da35-48d7-a028-4b7c614f5b86 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:31:15.633814Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21208" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4e070-predictor-6985d68b68-kls5q to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7a99dd1731 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: b458095c-ee0b-4e0f-9b1c-657373f46d3c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4e070-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7aa9ae3754 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: d670d6d0-0268-4e01-ac2b-0694c62b39cf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7ad83aae82 namespace: kserve-ci-e2e-test resourceVersion: "21276" uid: 9ff93679-01a9-4afe-9d81-982206a54757 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7ad9bba8cf namespace: kserve-ci-e2e-test resourceVersion: "21278" uid: 59648c66-6aba-45c2-b062-53bf88fb0365 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7adcdec684 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 7558085b-c8c8-49e2-b06f-f3b442b90f3c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7add5209d7 namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: 9bc4a66b-14ba-4eda-a68a-ed9d98f52c25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7add5cb250 namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 008950ad-db65-4d5b-949d-5dadb67c782d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7ae0d40a5b namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 95f971b5-9eb6-47f0-9987-e367e0ef55ac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:31:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:31:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:16Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7ae1462e11 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: 711b0a8e-609a-4231-bc37-67556c103091 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T10:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:40:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:33Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66d7af58b658a namespace: kserve-ci-e2e-test resourceVersion: "26026" uid: 0e1eee40-d884-4512-898c-0b94df49599b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66dfbcb28ef8c namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 032cb05b-2ca2-4cbf-b99b-93f662fc83e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66dfbcb29be28 namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: deb6b1c9-b08f-44b3-a876-54e8d1dbb023 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4e070-predictor-6985d68b68-kls5q namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 3116e4c0-2c47-4bab-8349-431399f7ac3b kind: Event lastTimestamp: "2026-07-28T10:40:33Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:33Z" name: success-200-isvc-4e070-predictor-6985d68b68-kls5q.18c66dfc6a0af4e6 namespace: kserve-ci-e2e-test resourceVersion: "26024" uid: 82228dd8-80e0-4bc7-abef-1407d6dd9d5b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4e070-predictor-6985d68b68 namespace: kserve-ci-e2e-test resourceVersion: "21192" uid: 8bcf891e-15a9-4107-b128-ac1ebf2626a3 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: 'Created pod: success-200-isvc-4e070-predictor-6985d68b68-kls5q' metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-4e070-predictor-6985d68b68.18c66d7a9956e4d1 namespace: kserve-ci-e2e-test resourceVersion: "21210" uid: a4cb8448-03b1-4198-b44b-b72bc8aaa3ce reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: a9e94e04-4098-493e-8ad0-a636528cbd25 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Scaled up replica set success-200-isvc-4e070-predictor-6985d68b68 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-4e070-predictor.18c66d7a978c680d namespace: kserve-ci-e2e-test resourceVersion: "21193" uid: 652ac00f-ca2a-4baa-9c14-b119dceac284 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 94c5ec24-bb0b-4582-bc86-68a40b8fbd43 kind: Event lastTimestamp: "2026-07-28T10:31:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:45Z" name: success-200-isvc-4e070-predictor.18c66d7e1631e766 namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: bf7a1b32-c565-45e5-b195-4c303846f5c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:31:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21190" uid: 94c5ec24-bb0b-4582-bc86-68a40b8fbd43 kind: Event lastTimestamp: "2026-07-28T10:31:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:31:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:45Z" name: success-200-isvc-4e070-predictor.18c66d7e16a4ce00 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: c5ccda4b-3d40-4835-bf70-34630f343b05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 94c5ec24-bb0b-4582-bc86-68a40b8fbd43 kind: Event lastTimestamp: "2026-07-28T10:32:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:30Z" name: success-200-isvc-4e070-predictor.18c66d85141789f0 namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: d388319e-f586-42f8-aa21-129368af0550 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:32:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4e070-predictor namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 94c5ec24-bb0b-4582-bc86-68a40b8fbd43 kind: Event lastTimestamp: "2026-07-28T10:32:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:32:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:30Z" name: success-200-isvc-4e070-predictor.18c66d851484f773 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 55e894e0-e192-4446-ac51-fab17af560db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 57b5e3d5-829a-40ff-944b-27d6fae9d38a kind: Event lastTimestamp: "2026-07-28T10:32:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:03Z" name: success-200-isvc-4e070.18c66d7a97dff75b namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: 05fc6cad-bffd-486a-aab7-ffc3a07f9e48 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4e070 namespace: kserve-ci-e2e-test resourceVersion: "21709" uid: 57b5e3d5-829a-40ff-944b-27d6fae9d38a kind: Event lastTimestamp: "2026-07-28T10:32:03Z" message: InferenceService [success-200-isvc-4e070] is Ready metadata: creationTimestamp: "2026-07-28T10:32:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:03Z" name: success-200-isvc-4e070.18c66d85b3dcca52 namespace: kserve-ci-e2e-test resourceVersion: "21710" uid: 14133e5b-4eaf-4d64-a964-1f2c85da0c23 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:19:20.466616Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-717c3-predictor-7db58fb9f4-htnjs to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:19:20Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd41693a797 namespace: kserve-ci-e2e-test resourceVersion: "14819" uid: 68c43f20-5508-467b-bb47-06b7429d3c9d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-717c3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:20Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd41f3eb152 namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: cd1f31d1-6592-431c-b437-f3a290a79938 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:19:21Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd4539f95ea namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: b0fad27f-d136-4451-8b2b-d01767489ee8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:21Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" metadata: creationTimestamp: "2026-07-28T10:19:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:21Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd454e646e1 namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: 9f8502f4-9c7e-4caf-bd10-ff75fc4bf19d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" in 15.37s (15.37s including waiting). Image size: 1334989926 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd7e909fe8d namespace: kserve-ci-e2e-test resourceVersion: "15103" uid: e39ba673-508a-442f-a9bb-c5ad8df80f69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd7ec7ea636 namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 50ada986-0397-4779-9a03-e3934188de21 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd7ece3e48c namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: a0917702-19f6-4751-9b9d-33018d31e21f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:36Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T10:19:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:36Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd7ecefec02 namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 2551f9ff-9dbe-4739-a7f4-e210fb0487e3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.439s (2.439s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd87e599497 namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 8c3d8ae0-8616-437c-80c2-db7a17d48f0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd88206bbec namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 322f19eb-f842-4d5d-8a47-cfba4106c844 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:19:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:19:39Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd8826cb3ba namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 3e1061d4-02ed-4322-a937-19f89021acaf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:19:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:20:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:15Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66cd883ae0977 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: d28820f4-cb53-45b6-97da-e3459be7f253 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66ce906073c64 namespace: kserve-ci-e2e-test resourceVersion: "15837" uid: 9582a868-1d82-4b64-93bc-410c6ad714d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66ce90607ceba namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 1e44c016-678a-46af-a9c2-82f9fb26d251 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: ebb3a5c4-6a18-4047-8d18-dc66c7f47f7d kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs.18c66ce90c0d3677 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: d7345ea0-809c-45f7-a7bb-9d4c0025b33f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-717c3-predictor-7db58fb9f4 namespace: kserve-ci-e2e-test resourceVersion: "14804" uid: ae04f445-c5e4-4f90-87d5-f29264b1bc52 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: 'Created pod: success-200-isvc-717c3-predictor-7db58fb9f4-htnjs' metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: success-200-isvc-717c3-predictor-7db58fb9f4.18c66cd41609481c namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 923c00b3-7cdf-4747-bb1e-6ea02a02ac7c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14802" uid: 0d8351a3-0c7f-404d-bf92-820924d9b7e7 kind: Event lastTimestamp: "2026-07-28T10:19:20Z" message: Scaled up replica set success-200-isvc-717c3-predictor-7db58fb9f4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:19:20Z" name: success-200-isvc-717c3-predictor.18c66cd413391eec namespace: kserve-ci-e2e-test resourceVersion: "14805" uid: de06b5da-c7a1-4096-bacf-611bb4d9b131 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: c551bd13-b831-4c03-bf77-9523b9813106 kind: Event lastTimestamp: "2026-07-28T10:20:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:20Z" name: success-200-isvc-717c3-predictor.18c66cd791c31b59 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 61ce7f0d-e8f6-49bc-9972-58bf2729b6ac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:19:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: c551bd13-b831-4c03-bf77-9523b9813106 kind: Event lastTimestamp: "2026-07-28T10:20:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:19:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:20Z" name: success-200-isvc-717c3-predictor.18c66cd792390d1d namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 03b838c9-0594-46c2-b266-34264d377839 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: c551bd13-b831-4c03-bf77-9523b9813106 kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: success-200-isvc-717c3-predictor.18c66ce58db6ff9a namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 700c28c9-00a4-4b0f-95f1-aac4f22ac0f7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-717c3-predictor namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: c551bd13-b831-4c03-bf77-9523b9813106 kind: Event lastTimestamp: "2026-07-28T10:20:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:20:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:35Z" name: success-200-isvc-717c3-predictor.18c66ce58e9e15a5 namespace: kserve-ci-e2e-test resourceVersion: "15636" uid: 03326b62-bc6f-4052-977e-db78b4cc9f05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T10:19:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-717c3 namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: b8343e38-55de-41bd-8c21-42fa670a7dda kind: Event lastTimestamp: "2026-07-28T10:20:25Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:19:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:25Z" name: success-200-isvc-717c3.18c66cd4133aaf81 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 42fd9330-0479-488f-92e4-10890543c5c2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15552" uid: b8343e38-55de-41bd-8c21-42fa670a7dda kind: Event lastTimestamp: "2026-07-28T10:20:25Z" message: InferenceService [success-200-isvc-717c3] is Ready metadata: creationTimestamp: "2026-07-28T10:20:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:25Z" name: success-200-isvc-717c3.18c66ce33f54f7fc namespace: kserve-ci-e2e-test resourceVersion: "15553" uid: fe519ee4-5e8b-4591-b717-eeed5c325c20 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:32:15.485043Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8d409-predictor-5f6cb868f4-8njwq to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d8889467357 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 0c72321e-8d45-445b-8297-3735601c66b5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8d409-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88938026d4 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: 161d89fd-0473-4e3d-a806-7f40f2dae056 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21915" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88c6ef8caf namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: e3175358-c1b2-4629-88c8-c1caa6e9d3fd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88c8a5227c namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: 127f4887-f3c1-4fe3-ad24-a99201b53ccb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88cce93906 namespace: kserve-ci-e2e-test resourceVersion: "22004" uid: 392e336c-9bbb-44f2-b312-4f794cbc867f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88cd5e3f40 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: 9b4f5a78-afa3-4ac8-977b-eb752c46c6aa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88cd67c67c namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 06ba9376-20b2-4ca3-808f-30d7570508ed reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88d0d57a57 namespace: kserve-ci-e2e-test resourceVersion: "22011" uid: 0c45282e-2992-4924-a2da-466ba81506e2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:16Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:32:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:16Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88d14ac670 namespace: kserve-ci-e2e-test resourceVersion: "22012" uid: 8d9d3b97-55d6-4d0e-ab81-86b1eca9e450 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:32:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:32:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:32:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:32:53Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66d88fc2d0776 namespace: kserve-ci-e2e-test resourceVersion: "22332" uid: d7352305-d366-44fc-a3ca-c16482cf8ac9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66e09af3d835b namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: 82d5dd33-58de-4afb-b231-f32be0899ddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 449aff97-4814-4b62-a575-b2095deefeaf kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq.18c66e09af3e5fd3 namespace: kserve-ci-e2e-test resourceVersion: "26526" uid: 305768b0-d2c1-4654-b931-0487576f3cfb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8d409-predictor-5f6cb868f4 namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 7361f41a-6c9e-42ff-ae4b-fbf885b01386 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'Created pod: success-200-isvc-8d409-predictor-5f6cb868f4-8njwq' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409-predictor-5f6cb868f4.18c66d8888d54f1d namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 03300ee3-1db8-4038-941d-e6f7949af793 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21879" uid: 8d93cb82-951f-44ee-a5d5-7cd98a73516d kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: Scaled up replica set success-200-isvc-8d409-predictor-5f6cb868f4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409-predictor.18c66d8886e4830d namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 36c83007-5c34-4cee-b799-dea614609b55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 143828be-f9ca-41b9-9cab-f5e9ec209c24 kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: success-200-isvc-8d409-predictor.18c66d8c05e3eadb namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: 14fa4da5-4029-4852-bbc0-5e16eeb689d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:32:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 143828be-f9ca-41b9-9cab-f5e9ec209c24 kind: Event lastTimestamp: "2026-07-28T10:32:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:32:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:32:45Z" name: success-200-isvc-8d409-predictor.18c66d8c064a48fa namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: c8d0afe2-6660-47b5-9083-46540e4c1e4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:33:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 143828be-f9ca-41b9-9cab-f5e9ec209c24 kind: Event lastTimestamp: "2026-07-28T10:33:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:33:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:30Z" name: success-200-isvc-8d409-predictor.18c66d93043fb3ef namespace: kserve-ci-e2e-test resourceVersion: "22659" uid: 7551562f-e009-49d8-8556-980c3a7725e6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:33:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8d409-predictor namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 143828be-f9ca-41b9-9cab-f5e9ec209c24 kind: Event lastTimestamp: "2026-07-28T10:33:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:33:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:33:30Z" name: success-200-isvc-8d409-predictor.18c66d9304a6a34a namespace: kserve-ci-e2e-test resourceVersion: "22661" uid: 28a8d028-1883-4432-8f1b-279bfaa2b7a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: e2ca6094-76ec-4521-8a29-ab787eea6cf5 kind: Event lastTimestamp: "2026-07-28T10:33:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:33:03Z" name: success-200-isvc-8d409.18c66d8886fdbe39 namespace: kserve-ci-e2e-test resourceVersion: "22422" uid: 4ff32187-046e-4b43-b7f2-d2af47fa5694 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: e2ca6094-76ec-4521-8a29-ab787eea6cf5 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'Failed to update status for InferenceService "success-200-isvc-8d409": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d409": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409.18c66d8887f9d244 namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 3c0a7fb4-7e37-400d-b0cc-68502f26a8ca reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:32:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: e2ca6094-76ec-4521-8a29-ab787eea6cf5 kind: Event lastTimestamp: "2026-07-28T10:32:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-8d409": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:32:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:32:15Z" name: success-200-isvc-8d409.18c66d8887fa3864 namespace: kserve-ci-e2e-test resourceVersion: "21903" uid: d920f540-89b4-4717-9e3c-04db497a0d3f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:33:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8d409 namespace: kserve-ci-e2e-test resourceVersion: "22416" uid: e2ca6094-76ec-4521-8a29-ab787eea6cf5 kind: Event lastTimestamp: "2026-07-28T10:33:03Z" message: InferenceService [success-200-isvc-8d409] is Ready metadata: creationTimestamp: "2026-07-28T10:33:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:33:03Z" name: success-200-isvc-8d409.18c66d93b8024538 namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: ef117f07-1067-4dc2-ba1a-df6386ef5a85 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:41:30.227058Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26543" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a647d-predictor-5474579676-s27ps to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09b279bd48 namespace: kserve-ci-e2e-test resourceVersion: "26551" uid: 5138aabb-b5d7-4559-af16-7691999b7c74 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26560" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09cc3ce25b namespace: kserve-ci-e2e-test resourceVersion: "26623" uid: 0dc17e6e-aa31-468b-b9fd-037d7ccd0d90 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09cdb2de33 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: d14ffaa9-bea4-4318-9a06-f4ed6b7ffdd2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09d1054b29 namespace: kserve-ci-e2e-test resourceVersion: "26630" uid: a870bedb-e6b0-46b4-94d4-7cb81029ad15 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09d1782074 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 567cc009-2ab9-4ff3-bd5f-53de9dfefb0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09d1827128 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 807666e6-a5de-4ad2-b67c-7feb46aa239e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09d4adf7bf namespace: kserve-ci-e2e-test resourceVersion: "26635" uid: 95d8d0a0-4e0b-4c56-9849-b45f019577c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e09d51fede5 namespace: kserve-ci-e2e-test resourceVersion: "26636" uid: 6adceaf9-01b9-48b2-a0a5-c34646b5c884 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:42:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:08Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66e0a3a3a39b2 namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: 20201b40-19f8-47c6-9e97-b5392083b332 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:58:49Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:49Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66efbaf3664a7 namespace: kserve-ci-e2e-test resourceVersion: "34307" uid: 9df40296-33df-43d6-9cd5-644eb1fe328b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a647d-predictor-5474579676-s27ps namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 1194a9bf-bc05-43b9-9b6b-354a76f2eab9 kind: Event lastTimestamp: "2026-07-28T10:58:49Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:49Z" name: success-200-isvc-a647d-predictor-5474579676-s27ps.18c66efbaf36e125 namespace: kserve-ci-e2e-test resourceVersion: "34308" uid: 7668ffa4-89df-4d59-a3f5-da14d0866575 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a647d-predictor-5474579676 namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 40b0160e-7ef0-49b2-bcf7-e00e789ec2e2 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'Created pod: success-200-isvc-a647d-predictor-5474579676-s27ps' metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor-5474579676.18c66e09b2035bff namespace: kserve-ci-e2e-test resourceVersion: "26546" uid: e3a30714-05c5-4e6f-8ae1-95fda587b07d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: edaea766-6a72-4407-976e-b00f66e7685d kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: Scaled up replica set success-200-isvc-a647d-predictor-5474579676 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-a647d-predictor.18c66e09afd0f821 namespace: kserve-ci-e2e-test resourceVersion: "26527" uid: 79714afc-0f8b-4850-912b-9d4ce7a02cfb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 4ab069e9-4ce4-4266-920a-c7c8c51c4cf2 kind: Event lastTimestamp: "2026-07-28T10:42:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:00Z" name: success-200-isvc-a647d-predictor.18c66e0d2e707373 namespace: kserve-ci-e2e-test resourceVersion: "27064" uid: ea2cee19-c51d-4d68-925a-d7390b4bf8c8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 4ab069e9-4ce4-4266-920a-c7c8c51c4cf2 kind: Event lastTimestamp: "2026-07-28T10:42:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:00Z" name: success-200-isvc-a647d-predictor.18c66e0d2ee34f8b namespace: kserve-ci-e2e-test resourceVersion: "27065" uid: a05468f1-9473-404d-9e12-69a49e2f86fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 4ab069e9-4ce4-4266-920a-c7c8c51c4cf2 kind: Event lastTimestamp: "2026-07-28T10:42:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:45Z" name: success-200-isvc-a647d-predictor.18c66e142c531997 namespace: kserve-ci-e2e-test resourceVersion: "27520" uid: 57af47b7-f486-4950-9017-067a03e0be49 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:42:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a647d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 4ab069e9-4ce4-4266-920a-c7c8c51c4cf2 kind: Event lastTimestamp: "2026-07-28T10:42:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:45Z" name: success-200-isvc-a647d-predictor.18c66e142cc45ff8 namespace: kserve-ci-e2e-test resourceVersion: "27522" uid: f88dee5e-f03a-4b47-a3a8-f2c647cc97fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T10:41:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "26503" uid: 1c329824-681f-4700-b35f-98b080da9142 kind: Event lastTimestamp: "2026-07-28T10:42:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:41:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:18Z" name: success-200-isvc-a647d.18c66e09afd753dc namespace: kserve-ci-e2e-test resourceVersion: "27226" uid: 260dd05d-ad77-4cd1-9264-095d367fae2c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a647d namespace: kserve-ci-e2e-test resourceVersion: "27224" uid: 1c329824-681f-4700-b35f-98b080da9142 kind: Event lastTimestamp: "2026-07-28T10:42:18Z" message: InferenceService [success-200-isvc-a647d] is Ready metadata: creationTimestamp: "2026-07-28T10:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:18Z" name: success-200-isvc-a647d.18c66e14f5ceb0a8 namespace: kserve-ci-e2e-test resourceVersion: "27225" uid: ea222022-d8b0-44f1-b88b-ff740520c020 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:30:05.395837Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a3f5a9950 namespace: kserve-ci-e2e-test resourceVersion: "20318" uid: 345cca3f-7ab5-4e0d-bc66-5c26600d7c23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a59bfee54 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: 5377af9e-3e73-4391-9e08-26dabfbcf9ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a5b25a2d4 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9121207d-cdbf-48b2-9057-a1bc0acbeb1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a5e1b310c namespace: kserve-ci-e2e-test resourceVersion: "20374" uid: ce6aa995-0867-4e12-9145-637526c19335 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a5e92a4e5 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 404fafaa-4dc0-40d6-b216-978678c2ab4b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a5ea6289c namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 1f668bdc-9b19-435c-942c-13ac388b8923 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a612676d1 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 34a806e0-21ac-45d1-a3a1-3ff9e5dfbcc3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a619b905b namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 1d841ca6-0c20-4c34-b291-2a423b24eda2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:30:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:42Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d6a96d5faf4 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 762d279b-3e0b-433a-89b7-3b1844040c2e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d7a966f1a0b namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 544f938c-0245-40ed-9387-f07f6d8c9a29 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:31:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:31:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:15Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d7a966fd9cb namespace: kserve-ci-e2e-test resourceVersion: "21188" uid: f9d7daab-a313-4591-86d9-4e9ab3fd13cd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:31:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: e3ba1909-9221-445a-8ed6-4b277f54fcb4 kind: Event lastTimestamp: "2026-07-28T10:31:17Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:31:17Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w.18c66d7b1f172708 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: f8571eab-65a0-441d-9f79-6f5fdc8e471b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-af89d-predictor-6fc7bd7dcb namespace: kserve-ci-e2e-test resourceVersion: "20302" uid: 167b7a0a-d63d-4e53-bf08-1d252f0cc44f kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: 'Created pod: success-200-isvc-af89d-predictor-6fc7bd7dcb-2989w' metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor-6fc7bd7dcb.18c66d6a3ece9fb8 namespace: kserve-ci-e2e-test resourceVersion: "20312" uid: 2a4e490f-9e6e-43a8-af87-5962f650e229 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20300" uid: 629156b1-d5fd-45fe-adde-3431376ef316 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Scaled up replica set success-200-isvc-af89d-predictor-6fc7bd7dcb from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-af89d-predictor.18c66d6a3b35b570 namespace: kserve-ci-e2e-test resourceVersion: "20303" uid: 96c919d8-7df9-4f32-b9c9-f7b0db037a5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 8afb6696-8140-4711-a1a5-eabf4b58b548 kind: Event lastTimestamp: "2026-07-28T10:30:20Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:20Z" name: success-200-isvc-af89d-predictor.18c66d6db9c43b2d namespace: kserve-ci-e2e-test resourceVersion: "20509" uid: fa9ee1ea-a432-426a-a5ac-653fd6d7a03a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20301" uid: 8afb6696-8140-4711-a1a5-eabf4b58b548 kind: Event lastTimestamp: "2026-07-28T10:30:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:30:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:30:20Z" name: success-200-isvc-af89d-predictor.18c66d6dba3471de namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: eafc8b2c-2d8c-4e23-8e1a-ff31fe2df599 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 8afb6696-8140-4711-a1a5-eabf4b58b548 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: success-200-isvc-af89d-predictor.18c66d7138d748f0 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: c332650f-329d-497a-94ac-09dc8c43cf84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T10:30:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-af89d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 8afb6696-8140-4711-a1a5-eabf4b58b548 kind: Event lastTimestamp: "2026-07-28T10:31:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:31:05Z" name: success-200-isvc-af89d-predictor.18c66d713921b6d6 namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: d54fec7b-0562-4257-8a33-50fa4b865e30 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-af89d namespace: kserve-ci-e2e-test resourceVersion: "20287" uid: 660dc151-3e4c-4b69-8b3a-de4c2380d5bd kind: Event lastTimestamp: "2026-07-28T10:30:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:53Z" name: success-200-isvc-af89d.18c66d6a3b3c8d3b namespace: kserve-ci-e2e-test resourceVersion: "20779" uid: e1b87a02-9d08-45cd-9454-874bf073c609 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-af89d namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 660dc151-3e4c-4b69-8b3a-de4c2380d5bd kind: Event lastTimestamp: "2026-07-28T10:30:53Z" message: InferenceService [success-200-isvc-af89d] is Ready metadata: creationTimestamp: "2026-07-28T10:30:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:30:53Z" name: success-200-isvc-af89d.18c66d755510ca6e namespace: kserve-ci-e2e-test resourceVersion: "20775" uid: facb05a9-75e4-4a85-8e9f-aec71033ed45 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:40:30.578202Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e803a-predictor-658895f866-4669k to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfbcf207617 namespace: kserve-ci-e2e-test resourceVersion: "25923" uid: 71ca418d-544d-4a75-bc58-6400fc3f64cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e803a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfbd9931e6f namespace: kserve-ci-e2e-test resourceVersion: "25971" uid: e9584faa-590c-4395-91ad-033a9655609a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25929" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc0c206411 namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: 20e243de-4c1e-4815-b999-09673f3a138c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc0d6dd794 namespace: kserve-ci-e2e-test resourceVersion: "25993" uid: bf8f15e5-96d8-409d-90fb-2f646f82955d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc10e88e6e namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: fb84a679-5027-4d12-9aec-eeedd54c8b02 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc11673407 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 76507be0-75d3-43a2-a17f-330a8d2ce510 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc1171f723 namespace: kserve-ci-e2e-test resourceVersion: "25996" uid: e8d6794e-6747-4502-ace0-1b4937de4f3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc1411e2a1 namespace: kserve-ci-e2e-test resourceVersion: "26000" uid: 43fb8618-55a4-4df6-8919-930eaaa8335a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:40:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:40:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:40:31Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc148cd5b2 namespace: kserve-ci-e2e-test resourceVersion: "26002" uid: fc1691af-dcbf-4163-a4ad-18f4d976f1df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:40:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:41:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:40:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:09Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66dfc6f87dd3e namespace: kserve-ci-e2e-test resourceVersion: "26314" uid: e0dd5d7e-f571-4930-aeea-474b8ee00a8c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66e0c273896c2 namespace: kserve-ci-e2e-test resourceVersion: "26787" uid: 17ec10ad-ba0d-49d5-8dea-eb4f599e060c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66e0c273f1456 namespace: kserve-ci-e2e-test resourceVersion: "26790" uid: 6969b264-b1f7-4fc1-83b7-53e36d478ebc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e803a-predictor-658895f866-4669k namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: d66a12c5-150d-46dc-b847-11572a6b16c1 kind: Event lastTimestamp: "2026-07-28T10:41:44Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:44Z" name: success-200-isvc-e803a-predictor-658895f866-4669k.18c66e0cf7d9a006 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: c113f0d4-0b00-4a77-891b-96ec17753697 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e803a-predictor-658895f866 namespace: kserve-ci-e2e-test resourceVersion: "25906" uid: e5a16d8f-5357-4581-9dd1-8effca2d627a kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'Created pod: success-200-isvc-e803a-predictor-658895f866-4669k' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a-predictor-658895f866.18c66dfbcead29c8 namespace: kserve-ci-e2e-test resourceVersion: "25920" uid: a1277901-0b62-4e96-96d5-792a74391112 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25901" uid: 328d2b2c-da2c-44a7-abd6-52d651d45402 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: Scaled up replica set success-200-isvc-e803a-predictor-658895f866 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a-predictor.18c66dfbcb20c4b0 namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 7d7d5401-b58b-4c8c-be09-7f9dbc753b14 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:40:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: 7e4a95cd-d14c-4b20-b4c9-cf418a17c905 kind: Event lastTimestamp: "2026-07-28T10:41:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:00Z" name: success-200-isvc-e803a-predictor.18c66dff4a0027a4 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 531d85cb-2ca2-4af3-9562-7565b79d0468 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:40:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "25904" uid: 7e4a95cd-d14c-4b20-b4c9-cf418a17c905 kind: Event lastTimestamp: "2026-07-28T10:41:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:40:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:00Z" name: success-200-isvc-e803a-predictor.18c66dff4a737b27 namespace: kserve-ci-e2e-test resourceVersion: "26244" uid: 5b975a12-3baa-4faa-a0a1-63b8fa8e7fed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: 7e4a95cd-d14c-4b20-b4c9-cf418a17c905 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-e803a-predictor.18c66e0649132b8b namespace: kserve-ci-e2e-test resourceVersion: "26614" uid: 76a69911-2b0e-4148-91f0-eddd963667bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e803a-predictor namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: 7e4a95cd-d14c-4b20-b4c9-cf418a17c905 kind: Event lastTimestamp: "2026-07-28T10:41:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:41:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:30Z" name: success-200-isvc-e803a-predictor.18c66e06498064e7 namespace: kserve-ci-e2e-test resourceVersion: "26615" uid: a43322a9-1014-46ce-8fc9-9a1358a88934 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 7702a57c-6908-4620-9ec2-8f4822b3cd66 kind: Event lastTimestamp: "2026-07-28T10:41:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:19Z" name: success-200-isvc-e803a.18c66dfbcb771f18 namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: 8b1d517e-2a25-439b-972f-2367ada7748e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 7702a57c-6908-4620-9ec2-8f4822b3cd66 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-e803a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e803a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a.18c66dfbcbd5ce05 namespace: kserve-ci-e2e-test resourceVersion: "25913" uid: fb1f35c2-4fcb-4422-9e42-2fe53f3c79b2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:40:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: 7702a57c-6908-4620-9ec2-8f4822b3cd66 kind: Event lastTimestamp: "2026-07-28T10:40:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e803a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:40:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:40:30Z" name: success-200-isvc-e803a.18c66dfbcbd62c63 namespace: kserve-ci-e2e-test resourceVersion: "25915" uid: 7116554d-ddd8-45da-aae5-cc1cc5590d0b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e803a namespace: kserve-ci-e2e-test resourceVersion: "26398" uid: 7702a57c-6908-4620-9ec2-8f4822b3cd66 kind: Event lastTimestamp: "2026-07-28T10:41:19Z" message: InferenceService [success-200-isvc-e803a] is Ready metadata: creationTimestamp: "2026-07-28T10:41:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:41:19Z" name: success-200-isvc-e803a.18c66e072b5fd404 namespace: kserve-ci-e2e-test resourceVersion: "26399" uid: d61a0434-6806-4816-a2da-f2941b7ee3fc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:20:50.442942Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce909957135 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 2b93264f-436c-4bda-8d86-ab4fcb2ba2e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e92a5-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce911d641bb namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 1bf4fd5e-d2da-46eb-824b-7bf0f03ee2f4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15878" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94731c75c namespace: kserve-ci-e2e-test resourceVersion: "15940" uid: f3fea884-c882-482d-91f5-ef5fe48bfd42 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce9487fbeab namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 6a93415d-5d26-4d74-95e5-4bb52d2d62e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94bdfcc47 namespace: kserve-ci-e2e-test resourceVersion: "15944" uid: 5f1701ba-22f8-4ad3-a295-6e2f8f4c40ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94c64019b namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: 727816f5-9a47-4940-ba78-1b65a7c6d431 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94c6e137e namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 834bda5c-83c0-461f-a098-942949b2b073 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94f024531 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: aaf55c0d-ce09-4d89-aac9-bd86f3c91aea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:20:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:20:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:51Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce94f65586d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 12f763c2-b978-4d9d-986c-280f9c91c370 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:21:28Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:20:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:28Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66ce9939fc74b namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: b97d6e1f-f6d7-4a59-b288-3efddcde64d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66d6a38483627 namespace: kserve-ci-e2e-test resourceVersion: "20292" uid: 105ac339-705c-4cd8-b8d8-e286ee6e9b37 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn namespace: kserve-ci-e2e-test resourceVersion: "15866" uid: ed33011d-7843-43e9-9b70-8db094db7646 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn.18c66d6a384a3f55 namespace: kserve-ci-e2e-test resourceVersion: "20293" uid: cdd15494-dfe6-42b7-84f0-ba02f4b498be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e92a5-predictor-5cdccb45b8 namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 61ac7328-a0d4-43cb-bc1f-12c0c26bcadc kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: 'Created pod: success-200-isvc-e92a5-predictor-5cdccb45b8-bnrjn' metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-e92a5-predictor-5cdccb45b8.18c66ce90928c281 namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 77948598-8526-4c20-b817-1ae09d15489c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15841" uid: fa122380-847b-4073-bd32-f065a2f7d206 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Scaled up replica set success-200-isvc-e92a5-predictor-5cdccb45b8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:50Z" name: success-200-isvc-e92a5-predictor.18c66ce9075de079 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: 858e2e77-9639-4c7b-8b75-615a396ccee6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: d6f51d58-0e29-4e85-b6d9-7297e765f58c kind: Event lastTimestamp: "2026-07-28T10:21:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:05Z" name: success-200-isvc-e92a5-predictor.18c66cec864f43b1 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: a88e9395-7cf0-4c10-a320-79764d259a6d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "15844" uid: d6f51d58-0e29-4e85-b6d9-7297e765f58c kind: Event lastTimestamp: "2026-07-28T10:21:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:21:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:05Z" name: success-200-isvc-e92a5-predictor.18c66cec86be5e6f namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: f85a89e3-5528-47a9-93dc-40dae69d05d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:21:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: d6f51d58-0e29-4e85-b6d9-7297e765f58c kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: success-200-isvc-e92a5-predictor.18c66cf0055f4aa0 namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 4e193931-075f-4c97-8333-9b3daed8a5f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T10:21:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e92a5-predictor namespace: kserve-ci-e2e-test resourceVersion: "16083" uid: d6f51d58-0e29-4e85-b6d9-7297e765f58c kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:21:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: success-200-isvc-e92a5-predictor.18c66cf005c8d6ca namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 6e16d808-2c53-40b6-b9ec-590559afc2a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "15821" uid: 47b616e0-08f8-4620-a3b2-8f1768f88824 kind: Event lastTimestamp: "2026-07-28T10:21:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:38Z" name: success-200-isvc-e92a5.18c66ce9075d2e32 namespace: kserve-ci-e2e-test resourceVersion: "16417" uid: 2f273599-22f7-495d-ad42-35dc24c5a7ae reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 47b616e0-08f8-4620-a3b2-8f1768f88824 kind: Event lastTimestamp: "2026-07-28T10:21:38Z" message: InferenceService [success-200-isvc-e92a5] is Ready metadata: creationTimestamp: "2026-07-28T10:21:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:38Z" name: success-200-isvc-e92a5.18c66cf44f6632e6 namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 57c04c3c-fab2-498a-80ed-9270bfe9b55e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:41:40.830038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26808" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv to ip-10-0-131-51.ec2.internal metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c2a76614c namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: d3bcf792-e7c5-4997-8892-2f4570035408 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f7711-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c3450e9d2 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 042a6e81-e5d5-4b9c-b225-00c8ebbf2d93 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:41:41Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c68c97374 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 9c39f55c-da63-4398-8b18-a37b9918cfc8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1813" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:41Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c6a36cf00 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: 5ff0fa7f-aa47-44ec-a88d-d7dd3e9e9620 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:41Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c6e758ebd namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 48a32d04-7ab1-4a98-9d57-9f940f369bcc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:41Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c6ef1f240 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 7d49333b-33ff-4a0c-ad9a-83a27bcfd3e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T10:41:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:41Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c6efb568f namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: c39200b6-6845-4a10-8899-1baec1d07783 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c71e83dd2 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 4ed5c14d-bd30-4933-9cd1-74aabdda9276 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:41:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:41:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:41:42Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0c725bf515 namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: f9f6a113-fbe3-4f5b-a098-306213f36cae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:42:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T10:41:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:19Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e0ccd4d2d17 namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: db31328f-7872-4dc1-9f84-fd5f908ab3dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:50:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:50:55Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e8d5dc657fa namespace: kserve-ci-e2e-test resourceVersion: "30990" uid: db9c3435-06ee-4b37-931f-7b428b6dd8bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:50:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: ea213dbf-7200-4c6d-8f12-78e23917b867 kind: Event lastTimestamp: "2026-07-28T10:50:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T10:50:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:50:55Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv.18c66e8d5dc6f152 namespace: kserve-ci-e2e-test resourceVersion: "30991" uid: 5fa91bf9-91e4-42dc-8354-bf5996f0dfd7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-131-51.ec2.internal source: component: kubelet host: ip-10-0-131-51.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7711-predictor-7b6f54fcd8 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: 1591b04f-61dc-4520-ab89-f900744ac414 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: 'Created pod: success-200-isvc-f7711-predictor-7b6f54fcd8-dkzjv' metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-f7711-predictor-7b6f54fcd8.18c66e0c29fef119 namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: b3f6c0eb-711a-4967-a14d-d96d0b6fb0d1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26786" uid: d4efb34c-fbc1-4ebf-830e-9011bbcd9262 kind: Event lastTimestamp: "2026-07-28T10:41:40Z" message: Scaled up replica set success-200-isvc-f7711-predictor-7b6f54fcd8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:41:40Z" name: success-200-isvc-f7711-predictor.18c66e0c27fc2916 namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 6120ef4c-9989-4aa8-9cd4-d02fbf1d68a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 18c368ed-3912-43f7-9d04-d39d910d91d4 kind: Event lastTimestamp: "2026-07-28T10:42:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:10Z" name: success-200-isvc-f7711-predictor.18c66e0fa6ce5c92 namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: 3161e7e2-1e06-4dca-a63c-4d60c422b4c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:41:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "26788" uid: 18c368ed-3912-43f7-9d04-d39d910d91d4 kind: Event lastTimestamp: "2026-07-28T10:42:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:41:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:10Z" name: success-200-isvc-f7711-predictor.18c66e0fa72aab27 namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 40d0de0c-d040-4604-b067-ee004be0f1a0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:42:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 18c368ed-3912-43f7-9d04-d39d910d91d4 kind: Event lastTimestamp: "2026-07-28T10:42:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:40Z" name: success-200-isvc-f7711-predictor.18c66e16a5163527 namespace: kserve-ci-e2e-test resourceVersion: "27459" uid: 39374ed8-e266-4773-a40a-4eddad04a138 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T10:42:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7711-predictor namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: 18c368ed-3912-43f7-9d04-d39d910d91d4 kind: Event lastTimestamp: "2026-07-28T10:42:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:42:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:40Z" name: success-200-isvc-f7711-predictor.18c66e16a583a4da namespace: kserve-ci-e2e-test resourceVersion: "27461" uid: beabe61f-5923-48c1-9590-f9436829529c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:41:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 0e76cf08-cc64-4cfa-b85d-91a695c02d79 kind: Event lastTimestamp: "2026-07-28T10:42:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T10:41:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:29Z" name: success-200-isvc-f7711.18c66e0c283412db namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: f943b44d-3744-488d-a663-f5a01206e792 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7711 namespace: kserve-ci-e2e-test resourceVersion: "27319" uid: 0e76cf08-cc64-4cfa-b85d-91a695c02d79 kind: Event lastTimestamp: "2026-07-28T10:42:29Z" message: InferenceService [success-200-isvc-f7711] is Ready metadata: creationTimestamp: "2026-07-28T10:42:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:29Z" name: success-200-isvc-f7711.18c66e1789471720 namespace: kserve-ci-e2e-test resourceVersion: "27320" uid: 78dc9026-a4fe-492a-801e-298f9a8cc0c9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:20:40.156717Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15688" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-717c3-b69fd89f-9mwps to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce6a47a1790 namespace: kserve-ci-e2e-test resourceVersion: "15693" uid: 0bae9d5c-3b48-46b4-91c4-f36ec1f3b671 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce6be31e110 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: eab7bf67-b534-4aea-b63a-c9204b4fbfad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce6bf84fbcc namespace: kserve-ci-e2e-test resourceVersion: "15711" uid: 20e8baf2-658b-4161-b9f1-46ad74c097d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:42Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" in 1.816s (1.816s including waiting). Image size: 218226791 bytes.' metadata: creationTimestamp: "2026-07-28T10:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:42Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce72bd246a1 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: b2ea269a-d837-43a8-93c5-3326f7340668 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:42Z" message: 'Created container: switch-graph-717c3' metadata: creationTimestamp: "2026-07-28T10:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:42Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce72f079db0 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: c1dd96e7-9aca-4440-b5c1-8a80ce1529d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:42Z" message: Started container switch-graph-717c3 metadata: creationTimestamp: "2026-07-28T10:20:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:42Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce72f928c2f namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 28a99b74-9918-42ba-958e-22c8ff8f7280 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:20:50Z" message: Stopping container switch-graph-717c3 metadata: creationTimestamp: "2026-07-28T10:20:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:20:50Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce8fef6e95f namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: 72aed0d1-be17-4ba7-bf9b-3d55717fb6b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:20:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-717c3} kind: Pod name: switch-graph-717c3-b69fd89f-9mwps namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 484d7d3d-b078-424d-8463-35b214f2adb0 kind: Event lastTimestamp: "2026-07-28T10:21:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:20:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:19Z" name: switch-graph-717c3-b69fd89f-9mwps.18c66ce9f653f447 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 319368fd-8273-4f83-bd08-32e1bb7be960 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-717c3-b69fd89f namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: e75742cc-c107-43cf-99c2-2b9dde6a55ec kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: 'Created pod: switch-graph-717c3-b69fd89f-9mwps' metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3-b69fd89f.18c66ce6a3e3ab04 namespace: kserve-ci-e2e-test resourceVersion: "15689" uid: 47875178-9737-4085-ae1b-3524abc9ef6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15683" uid: 36720bbb-e83a-46a9-ae40-102c05d9a611 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: Scaled up replica set switch-graph-717c3-b69fd89f from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3.18c66ce6a30fcc52 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 52076593-684e-4b79-99f9-b4dba7e3e2c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 7b6ec455-6971-44ce-b918-e74270a9bca0 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: InferenceGraph [switch-graph-717c3] is Ready metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3.18c66ce6a84ccfff namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: a65e35a6-deaf-4e95-a701-2b5aec63a1ca reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 7b6ec455-6971-44ce-b918-e74270a9bca0 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: 'Failed to update status for InferenceGraph "switch-graph-717c3": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-717c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3.18c66ce6aa94dd92 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: e91346cd-f315-482f-9829-22051fd7e46f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:20:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-717c3 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 7b6ec455-6971-44ce-b918-e74270a9bca0 kind: Event lastTimestamp: "2026-07-28T10:20:40Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-717c3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T10:20:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:20:40Z" name: switch-graph-717c3.18c66ce6aa952268 namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: 76dd5926-df0d-4275-ad2e-8ca9265a4905 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:42:30.269880Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-a647d-7bc89888f9-ftrq7 to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:42:30Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17ad4e8c45 namespace: kserve-ci-e2e-test resourceVersion: "27345" uid: bd1b752a-41f4-47bd-9631-915576c3f450 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:30Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-a647d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:30Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17b3d5fd2f namespace: kserve-ci-e2e-test resourceVersion: "27358" uid: d0a29798-b093-4946-868a-64d1140efbd4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27351" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:42:31Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:42:31Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17d8e42fe7 namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 0dbb7567-9fe2-4130-9a79-38b083f9bd19 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a647d} kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:42:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:31Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17da298038 namespace: kserve-ci-e2e-test resourceVersion: "27374" uid: 872bfcdc-930e-44a5-9096-9e8a6ad64d06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a647d} kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:42:31Z" message: 'Created container: switch-graph-a647d' metadata: creationTimestamp: "2026-07-28T10:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:31Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17dd3f4d4b namespace: kserve-ci-e2e-test resourceVersion: "27376" uid: 2846deeb-dbe1-42f7-8139-81848e34fc88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a647d} kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:42:31Z" message: Started container switch-graph-a647d metadata: creationTimestamp: "2026-07-28T10:42:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:42:31Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66e17ddb733bd namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: fc795b22-1628-4322-9158-0519a386a6f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:58:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a647d} kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:58:49Z" message: Stopping container switch-graph-a647d metadata: creationTimestamp: "2026-07-28T10:58:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:58:49Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66efba570ecf5 namespace: kserve-ci-e2e-test resourceVersion: "34290" uid: e1d41a80-0133-4b40-b376-e674a9893068 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T10:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-a647d} kind: Pod name: switch-graph-a647d-7bc89888f9-ftrq7 namespace: kserve-ci-e2e-test resourceVersion: "27341" uid: af518c78-57e4-45a2-bad3-4c7275a8e597 kind: Event lastTimestamp: "2026-07-28T10:59:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:59:12Z" name: switch-graph-a647d-7bc89888f9-ftrq7.18c66efc52822933 namespace: kserve-ci-e2e-test resourceVersion: "34529" uid: 46b1ddbe-615e-44d8-8f5d-e05694caeec8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-a647d-7bc89888f9 namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 3559e674-f7ec-4656-b64d-041abd38f145 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: 'Created pod: switch-graph-a647d-7bc89888f9-ftrq7' metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:30Z" name: switch-graph-a647d-7bc89888f9.18c66e17acd6d3b5 namespace: kserve-ci-e2e-test resourceVersion: "27340" uid: eb0f5c0a-ae4d-4531-a955-595a958f063c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27332" uid: 29481736-4534-470d-a517-41ec3ce3dbaa kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: Scaled up replica set switch-graph-a647d-7bc89888f9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:30Z" name: switch-graph-a647d.18c66e17ac0db4a6 namespace: kserve-ci-e2e-test resourceVersion: "27337" uid: d9e86906-471f-4d1e-84b0-f52dbb348b77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27354" uid: 97587d0a-7447-4904-9966-cf3d31000e69 kind: Event lastTimestamp: "2026-07-28T10:42:30Z" message: InferenceGraph [switch-graph-a647d] is Ready metadata: creationTimestamp: "2026-07-28T10:42:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:42:30Z" name: switch-graph-a647d.18c66e17b1a3b664 namespace: kserve-ci-e2e-test resourceVersion: "27357" uid: 17d0f035-8346-4933-91cd-7762a40ee9aa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: cd9631f3-a618-4587-9c2e-4a9973fadae0 kind: Event lastTimestamp: "2026-07-28T10:42:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:45Z" name: switch-graph-a647d.18c66e1b2a8087d8 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: bfdcb131-888f-4a39-8dd3-0d2f13b4bc46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:42:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: cd9631f3-a618-4587-9c2e-4a9973fadae0 kind: Event lastTimestamp: "2026-07-28T10:42:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:42:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:42:45Z" name: switch-graph-a647d.18c66e1b2adfa1e0 namespace: kserve-ci-e2e-test resourceVersion: "27524" uid: 5c506121-5e86-45f9-8ce3-ba6ce5782e89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: cd9631f3-a618-4587-9c2e-4a9973fadae0 kind: Event lastTimestamp: "2026-07-28T10:43:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:43:00Z" name: switch-graph-a647d.18c66e1ea9b9660f namespace: kserve-ci-e2e-test resourceVersion: "27640" uid: fe16d95b-7a39-4969-a15b-b4fb77d523cf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:43:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-a647d namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: cd9631f3-a618-4587-9c2e-4a9973fadae0 kind: Event lastTimestamp: "2026-07-28T10:43:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:43:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:43:00Z" name: switch-graph-a647d.18c66e1eaa046618 namespace: kserve-ci-e2e-test resourceVersion: "27642" uid: 306bc9bf-4679-4283-b6bd-42be3448761c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T10:21:50.503120Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16686" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e92a5-6d7f7969cc-8q5zh to ip-10-0-140-175.ec2.internal metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T10:21:50Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf7057308cc namespace: kserve-ci-e2e-test resourceVersion: "16693" uid: e9126211-f89e-41e0-897c-feaf628149ce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84d4864675-w27z8 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:21:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e92a5-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:50Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf7136bb06a namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 20716edb-638f-4d5a-8567-89d34995289e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:51Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16697" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:21:51Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T10:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T10:21:51Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf74318e6d5 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 3473ff66-e7b8-44aa-9bcb-dd4114cad282 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e92a5} kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:21:51Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:bcb85ef6ed85b7da131182bb5f673741b67400a2637d62d1dcee3f3c438674f0" already present on machine metadata: creationTimestamp: "2026-07-28T10:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:51Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf7445d7af6 namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: 6475b092-9e27-43d2-ba5e-429525c99650 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e92a5} kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:21:51Z" message: 'Created container: switch-graph-e92a5' metadata: creationTimestamp: "2026-07-28T10:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:51Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf74791e6d1 namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 6947141d-ab4a-489e-99d1-1fd899300a20 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e92a5} kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:21:51Z" message: Started container switch-graph-e92a5 metadata: creationTimestamp: "2026-07-28T10:21:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:21:51Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66cf7481fe70b namespace: kserve-ci-e2e-test resourceVersion: "16721" uid: f59a8293-2ce1-420e-a1b7-66d2af53807b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:30:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e92a5} kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:30:05Z" message: Stopping container switch-graph-e92a5 metadata: creationTimestamp: "2026-07-28T10:30:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:05Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66d6a301c03cd namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: 7950f39e-2fe0-4bf8-9b9a-86796941925a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T10:30:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e92a5} kind: Pod name: switch-graph-e92a5-6d7f7969cc-8q5zh namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: a8b66e0e-90c6-4d4f-83ba-d7f49ea327d2 kind: Event lastTimestamp: "2026-07-28T10:30:32Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T10:30:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T10:30:32Z" name: switch-graph-e92a5-6d7f7969cc-8q5zh.18c66d6acb9e4d3a namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: dacd5268-6645-47a6-827f-b4a59ce3e131 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-175.ec2.internal source: component: kubelet host: ip-10-0-140-175.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e92a5-6d7f7969cc namespace: kserve-ci-e2e-test resourceVersion: "16682" uid: 11c7b040-8c32-4ce4-a9cf-0b7fc9a9b794 kind: Event lastTimestamp: "2026-07-28T10:21:50Z" message: 'Created pod: switch-graph-e92a5-6d7f7969cc-8q5zh' metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:50Z" name: switch-graph-e92a5-6d7f7969cc.18c66cf704fbaad5 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 6003dff8-fd71-4890-b950-ec6960ae5655 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16680" uid: 55b8565c-913b-4511-be0e-63f8fcf4cc72 kind: Event lastTimestamp: "2026-07-28T10:21:50Z" message: Scaled up replica set switch-graph-e92a5-6d7f7969cc from 0 to 1 metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:21:50Z" name: switch-graph-e92a5.18c66cf7044174de namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 8cf992a5-b22e-4f0d-8d44-3030edbcf523 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:21:50Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16702" uid: def42dc8-4606-4d91-9262-93b16fe74b1f kind: Event lastTimestamp: "2026-07-28T10:21:50Z" message: InferenceGraph [switch-graph-e92a5] is Ready metadata: creationTimestamp: "2026-07-28T10:21:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T10:21:50Z" name: switch-graph-e92a5.18c66cf709a9e905 namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: 1d44f82d-340a-4528-95d8-fed0d4df3148 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 21ee83ba-7acf-4931-a130-f0164cf592a3 kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: switch-graph-e92a5.18c66cfa8288b729 namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: ef4164a4-23f9-46de-9d1a-a5c6899f9aec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16681" uid: 21ee83ba-7acf-4931-a130-f0164cf592a3 kind: Event lastTimestamp: "2026-07-28T10:22:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T10:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:05Z" name: switch-graph-e92a5.18c66cfa82f9a768 namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: a5121117-e7c2-4f0a-9b41-b7a899ab91ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 21ee83ba-7acf-4931-a130-f0164cf592a3 kind: Event lastTimestamp: "2026-07-28T10:22:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:20Z" name: switch-graph-e92a5.18c66cfe018ca0bc namespace: kserve-ci-e2e-test resourceVersion: "16964" uid: a2ebe220-1e53-4835-bdf6-b7f758e5236b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T10:22:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e92a5 namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 21ee83ba-7acf-4931-a130-f0164cf592a3 kind: Event lastTimestamp: "2026-07-28T10:22:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T10:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T10:22:20Z" name: switch-graph-e92a5.18c66cfe01db7eea namespace: kserve-ci-e2e-test resourceVersion: "16966" uid: 47d626f9-6517-4812-bde1-0b1918086f5c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34561"