--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:03:18.033289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-b2e72-6778fd8957-k8n72 to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:03:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:03:18Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7aafd1386476 namespace: kserve-ci-e2e-test resourceVersion: "22365" uid: 6c40d561-f9b2-4842-8cef-262cb8ffbac0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:18Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:03:18Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-b2e72-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:18Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7aafdd3a6e00 namespace: kserve-ci-e2e-test resourceVersion: "22375" uid: 1332ce1c-efce-44c4-88de-c58ef5389ae2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:19Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22368" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:03:19Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:03:19Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7ab00dfae680 namespace: kserve-ci-e2e-test resourceVersion: "22387" uid: a897245c-841a-460c-b0d8-09a6a63c1c46 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e72} kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:03:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:19Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7ab00f1a0194 namespace: kserve-ci-e2e-test resourceVersion: "22389" uid: 3726318f-a831-4096-b4a1-3d33a85453f7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e72} kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:03:19Z" message: 'Created container: ensemble-graph-b2e72' metadata: creationTimestamp: "2026-07-02T13:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:19Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7ab01230510a namespace: kserve-ci-e2e-test resourceVersion: "22390" uid: 3af346db-37b7-44a2-93a2-e72b06a3ccde reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e72} kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:03:19Z" message: Started container ensemble-graph-b2e72 metadata: creationTimestamp: "2026-07-02T13:03:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:19Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7ab012bcff55 namespace: kserve-ci-e2e-test resourceVersion: "22391" uid: 1f14adec-2603-4321-8dfc-45871917bdce reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e72} kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:11:32Z" message: Stopping container ensemble-graph-b2e72 metadata: creationTimestamp: "2026-07-02T13:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:32Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7b2301ae9f22 namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 0c5e362d-23a9-4133-be4d-22d1d594c465 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-b2e72} kind: Pod name: ensemble-graph-b2e72-6778fd8957-k8n72 namespace: kserve-ci-e2e-test resourceVersion: "22363" uid: 3b785aef-0283-4aae-b551-320bf2e51374 kind: Event lastTimestamp: "2026-07-02T13:12:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:00Z" name: ensemble-graph-b2e72-6778fd8957-k8n72.18be7b23927f2ea0 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 726fa602-28d5-49f5-a1a0-b59ead56e545 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-b2e72-6778fd8957 namespace: kserve-ci-e2e-test resourceVersion: "22355" uid: 6c414fd3-8e1d-4cf6-b2a8-fe9efd027b09 kind: Event lastTimestamp: "2026-07-02T13:03:18Z" message: 'Created pod: ensemble-graph-b2e72-6778fd8957-k8n72' metadata: creationTimestamp: "2026-07-02T13:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:18Z" name: ensemble-graph-b2e72-6778fd8957.18be7aafd02b9b3f namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: e22cd006-33fa-40a4-b1aa-cb90d634d82a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22353" uid: e952fcfe-eb7c-4d4d-9abd-6b30c0a2102a kind: Event lastTimestamp: "2026-07-02T13:03:17Z" message: Scaled up replica set ensemble-graph-b2e72-6778fd8957 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:18Z" name: ensemble-graph-b2e72.18be7aafcf348b13 namespace: kserve-ci-e2e-test resourceVersion: "22357" uid: 54db55b0-7409-482c-a9e1-6f719fedcf60 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:18Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22370" uid: bbaa7833-f8e3-49f6-8233-f44c353efe51 kind: Event lastTimestamp: "2026-07-02T13:03:18Z" message: InferenceGraph [ensemble-graph-b2e72] is Ready metadata: creationTimestamp: "2026-07-02T13:03:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:18Z" name: ensemble-graph-b2e72.18be7aafd8277415 namespace: kserve-ci-e2e-test resourceVersion: "22373" uid: bb8029a2-4308-45f0-bc11-d0499661bdcf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 4bda304c-54df-4e43-8755-9bcede1de967 kind: Event lastTimestamp: "2026-07-02T13:03:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:33Z" name: ensemble-graph-b2e72.18be7ab34db5bbf8 namespace: kserve-ci-e2e-test resourceVersion: "22508" uid: 7ea7f8dc-c232-4bd2-b4ed-a55e169e889f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 4bda304c-54df-4e43-8755-9bcede1de967 kind: Event lastTimestamp: "2026-07-02T13:03:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:33Z" name: ensemble-graph-b2e72.18be7ab34e1356bb namespace: kserve-ci-e2e-test resourceVersion: "22509" uid: fd7e92c6-d84d-48fa-8f12-ed685f23f215 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22507" uid: 4bda304c-54df-4e43-8755-9bcede1de967 kind: Event lastTimestamp: "2026-07-02T13:03:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:48Z" name: ensemble-graph-b2e72.18be7ab6ccf53bc1 namespace: kserve-ci-e2e-test resourceVersion: "22624" uid: 1c0344d4-211f-456e-b332-3c0c3a67d8fa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22507" uid: 4bda304c-54df-4e43-8755-9bcede1de967 kind: Event lastTimestamp: "2026-07-02T13:03:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:03:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:48Z" name: ensemble-graph-b2e72.18be7ab6cd684467 namespace: kserve-ci-e2e-test resourceVersion: "22626" uid: 84229827-1e88-418c-b140-6a84d23d97a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:01:57.711961Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21309" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c0346-6b7fdbb874-w6rwf to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:01:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:01:57Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9d1db22fa7 namespace: kserve-ci-e2e-test resourceVersion: "21314" uid: ca143016-9c24-44b5-9312-3d2950999010 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:01:58Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9d36c975e1 namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: df1988c9-a5cf-4563-8fb6-877199038032 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c0346} kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:58Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9d381a9025 namespace: kserve-ci-e2e-test resourceVersion: "21333" uid: 1304faf9-6a2e-4c6e-b373-6455033ac533 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c0346} kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: 'Created container: ensemble-graph-c0346' metadata: creationTimestamp: "2026-07-02T13:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:58Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9d3b282cef namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 9e2d1413-65ea-4325-b41d-6a9bc987875f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c0346} kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:01:58Z" message: Started container ensemble-graph-c0346 metadata: creationTimestamp: "2026-07-02T13:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:58Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9d3b956435 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 388bb408-4143-4b8b-a002-12060fce0116 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c0346} kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: Stopping container ensemble-graph-c0346 metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:07Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9f767dfaae namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 7879af3e-af86-4125-b7dd-c97351ed05b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c0346} kind: Pod name: ensemble-graph-c0346-6b7fdbb874-w6rwf namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: fdcc1ce2-39d9-48d5-a5e4-da21c88a84e1 kind: Event lastTimestamp: "2026-07-02T13:02:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:34Z" name: ensemble-graph-c0346-6b7fdbb874-w6rwf.18be7a9ff4680c9e namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: e6a74529-70d6-487f-b1eb-1c26f992a575 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c0346-6b7fdbb874 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: b0742d7b-a737-4cad-b2c7-e4e211c57641 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: 'Created pod: ensemble-graph-c0346-6b7fdbb874-w6rwf' metadata: creationTimestamp: "2026-07-02T13:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: ensemble-graph-c0346-6b7fdbb874.18be7a9d1d06babf namespace: kserve-ci-e2e-test resourceVersion: "21311" uid: 5301766a-3c96-43c2-aee5-94380b770eb1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c0346 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: cd19df38-eb01-4c8a-bed2-3cec3f85c5c7 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: Scaled up replica set ensemble-graph-c0346-6b7fdbb874 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: ensemble-graph-c0346.18be7a9d1bc6379c namespace: kserve-ci-e2e-test resourceVersion: "21308" uid: 10e14003-201f-4a80-9853-5b3fae77bcc9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c0346 namespace: kserve-ci-e2e-test resourceVersion: "21322" uid: 91333679-3314-44e2-9af0-417b068b9721 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: InferenceGraph [ensemble-graph-c0346] is Ready metadata: creationTimestamp: "2026-07-02T13:01:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:57Z" name: ensemble-graph-c0346.18be7a9d2210f0cd namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: 60547169-5201-481d-9ef4-3dd9b952f500 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:51:42.904509Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16213" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-04e3b-predictor-5756f97b8-66z92 to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0df8510aa5 namespace: kserve-ci-e2e-test resourceVersion: "16217" uid: 2e0808aa-950d-4a08-bf00-6ce751be4b9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e126c6649 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 35a48946-40c9-4db0-a9d0-862dc8f5302e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e13e6c86f namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 6d2b0139-d32f-4e41-82a2-9f980e22dd8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e16450304 namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: 9448b989-6a5a-49ee-89d9-113852680a17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e16d1f0ac namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 8eae4b9b-7a1c-4e0d-bd0a-b49cffba6445 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e16da2726 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 98c9d388-6383-4d68-a97f-852060a0277b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e1a025ece namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 6411e6f9-1d48-4639-bd17-9b9725e935b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e1a6f9a01 namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 3a322919-ff56-4c0f-b988-43e2ca1c2a2f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T12:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T13:01:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:00Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a0e4a43a2e6 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 522ea34b-3df7-4ee8-ba41-24dc752f393c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:57Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a8f325a693e namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 6da1eb88-c4d1-4abc-809b-b6c939cddb14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:57Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a8f325b8191 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: fb795687-e71a-4ad2-a2b1-6be444a198fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-04e3b-predictor-5756f97b8-66z92 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: f41086ca-3185-4e23-9d33-bb1539172af1 kind: Event lastTimestamp: "2026-07-02T13:01:00Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:00Z" name: error-404-isvc-04e3b-predictor-5756f97b8-66z92.18be7a8fbea19c7c namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: 5ecf8b69-17ab-4a9f-bfad-86caf1d3296a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-04e3b-predictor-5756f97b8 namespace: kserve-ci-e2e-test resourceVersion: "16203" uid: e5f745b8-78a2-405c-8ba7-b1d4b5f22311 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Created pod: error-404-isvc-04e3b-predictor-5756f97b8-66z92' metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-04e3b-predictor-5756f97b8.18be7a0df7f6ab48 namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: cd3e847e-4610-4058-aa9e-cc49d5eec1a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 6a732292-c2a9-43cb-8fbf-11697df7f65f kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Scaled up replica set error-404-isvc-04e3b-predictor-5756f97b8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-04e3b-predictor.18be7a0df67e3b76 namespace: kserve-ci-e2e-test resourceVersion: "16204" uid: b0ec5740-18bb-48d7-a292-5a671a298539 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 947c771c-650d-4c81-92c2-f23ef7a01e19 kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: error-404-isvc-04e3b-predictor.18be7a1175111dd1 namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 4dd25abf-555b-44f4-8c1a-0fa73705a50b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 947c771c-650d-4c81-92c2-f23ef7a01e19 kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: error-404-isvc-04e3b-predictor.18be7a11759a4d3c namespace: kserve-ci-e2e-test resourceVersion: "17041" uid: 4c7f62e6-d273-4b28-bbdc-6a9a0071eb9d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 947c771c-650d-4c81-92c2-f23ef7a01e19 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: error-404-isvc-04e3b-predictor.18be7a14f43d9d5d namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 2cae05f9-b69e-4659-8c57-3a1ee4ae8323 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16420" uid: 947c771c-650d-4c81-92c2-f23ef7a01e19 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: error-404-isvc-04e3b-predictor.18be7a14f4971a0b namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 407b8ec4-e11c-400f-bef7-86bf848328e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: bd8200b7-6c12-4397-80cc-a99b40c8862c kind: Event lastTimestamp: "2026-07-02T12:52:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:30Z" name: error-404-isvc-04e3b.18be7a0df67db7b0 namespace: kserve-ci-e2e-test resourceVersion: "16912" uid: 5ab9da22-6348-4db2-94ca-3e379b7a77e7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: bd8200b7-6c12-4397-80cc-a99b40c8862c kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Failed to update status for InferenceService "error-404-isvc-04e3b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04e3b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-04e3b.18be7a0df6f79471 namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 6116e4f2-9ad0-4bb2-b8b3-18c4b92206f7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: bd8200b7-6c12-4397-80cc-a99b40c8862c kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-04e3b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-04e3b.18be7a0df6f7e99d namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: d71f8b80-efb8-4711-941a-101883d7a5fa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: bd8200b7-6c12-4397-80cc-a99b40c8862c kind: Event lastTimestamp: "2026-07-02T12:52:30Z" message: InferenceService [error-404-isvc-04e3b] is Ready metadata: creationTimestamp: "2026-07-02T12:52:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:30Z" name: error-404-isvc-04e3b.18be7a1905c93002 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: d84c3570-5281-4a8e-ba33-f3bd35ae386c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:03:08.049413Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22216" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad7e228e74 namespace: kserve-ci-e2e-test resourceVersion: "22228" uid: 36b7f4b3-030b-4d0d-9b6d-57e46db8a8a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad97a04fc9 namespace: kserve-ci-e2e-test resourceVersion: "22247" uid: 3ec0fd38-d979-402b-83c5-932a895131d4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad99052414 namespace: kserve-ci-e2e-test resourceVersion: "22249" uid: b4ed6d5e-3f00-4590-a2c6-2f08eba330dd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad9c6f50e3 namespace: kserve-ci-e2e-test resourceVersion: "22250" uid: 3cab23bc-5625-415c-a57d-d4ae442eedd9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad9cdcd9d1 namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 9f732344-5217-46b5-b57b-0029a3c68d57 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad9ce66c01 namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: 932f7d77-cd5f-44f4-8077-72042a491779 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aad9feb9cdc namespace: kserve-ci-e2e-test resourceVersion: "22253" uid: c113f288-b93b-4f80-a361-882072325fe3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aada05cd45d namespace: kserve-ci-e2e-test resourceVersion: "22254" uid: 0dc4bff5-ca10-4ea1-93d9-684b2ae26cff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:25Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7aadb58a3e24 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: f332a496-6140-4fb8-bdb9-eeca4f48749c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7b2ec2c21a51 namespace: kserve-ci-e2e-test resourceVersion: "26801" uid: 16583876-f1e3-476f-8c02-c9b40c857fd0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7b2ec2c2af84 namespace: kserve-ci-e2e-test resourceVersion: "26802" uid: 88228428-01ef-4c00-86ed-18301921a22d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8 namespace: kserve-ci-e2e-test resourceVersion: "22223" uid: 36d5af22-ebaa-484e-a59f-6f404f343543 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8.18be7b2f2a2f705d namespace: kserve-ci-e2e-test resourceVersion: "26840" uid: 035a5075-8d1d-4f8e-8c1c-234dfc1f9e6d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5b6c7-predictor-757c7c7cd namespace: kserve-ci-e2e-test resourceVersion: "22203" uid: a3573cb2-7bdc-4475-8abc-04c88a3fddf5 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'Created pod: error-404-isvc-5b6c7-predictor-757c7c7cd-26lr8' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor-757c7c7cd.18be7aad7d966569 namespace: kserve-ci-e2e-test resourceVersion: "22220" uid: 93d0c148-6037-442c-913f-09d379549709 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 8868fa3d-b212-4503-8f9b-9700cea1d2d3 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Scaled up replica set error-404-isvc-5b6c7-predictor-757c7c7cd from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7-predictor.18be7aad7bf203df namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: ea6b3121-68ac-4c37-a270-d75f884cdb9c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:03:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: a03dbf8d-ab26-43c7-80ce-512b2693adaa kind: Event lastTimestamp: "2026-07-02T13:03:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:53Z" name: error-404-isvc-5b6c7-predictor.18be7ab0fa7f4aac namespace: kserve-ci-e2e-test resourceVersion: "22670" uid: 1a7c389f-da5e-41d2-87ac-03e19f4730ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:03:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: a03dbf8d-ab26-43c7-80ce-512b2693adaa kind: Event lastTimestamp: "2026-07-02T13:03:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:53Z" name: error-404-isvc-5b6c7-predictor.18be7ab0fb1cb2ea namespace: kserve-ci-e2e-test resourceVersion: "22671" uid: 7a3b72ad-3c11-446f-bf10-33e4196ab24a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: a03dbf8d-ab26-43c7-80ce-512b2693adaa kind: Event lastTimestamp: "2026-07-02T13:04:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:23Z" name: error-404-isvc-5b6c7-predictor.18be7ab479fdc228 namespace: kserve-ci-e2e-test resourceVersion: "22952" uid: 852d7511-9c05-4dc7-b242-29efa73b723a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22427" uid: a03dbf8d-ab26-43c7-80ce-512b2693adaa kind: Event lastTimestamp: "2026-07-02T13:04:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:23Z" name: error-404-isvc-5b6c7-predictor.18be7ab47a9102e5 namespace: kserve-ci-e2e-test resourceVersion: "22953" uid: fa4ef46d-902e-42a4-8f4b-1c4f7f0b6209 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: e021f0ed-bdb8-4b13-b2b1-a61d2f81e4bb kind: Event lastTimestamp: "2026-07-02T13:03:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:55Z" name: error-404-isvc-5b6c7.18be7aad7c8d6d16 namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: fe5b120a-8c67-4675-ac2a-7012fdd98fd5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: e021f0ed-bdb8-4b13-b2b1-a61d2f81e4bb kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-5b6c7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5b6c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7.18be7aad7d253ba3 namespace: kserve-ci-e2e-test resourceVersion: "22215" uid: 8c4b004f-b17a-469c-b991-a70ab10f9e79 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22180" uid: e021f0ed-bdb8-4b13-b2b1-a61d2f81e4bb kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5b6c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:08Z" name: error-404-isvc-5b6c7.18be7aad7d25aaa4 namespace: kserve-ci-e2e-test resourceVersion: "22219" uid: 881245f9-442e-465c-b1d3-ea7de9af8ea9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22692" uid: e021f0ed-bdb8-4b13-b2b1-a61d2f81e4bb kind: Event lastTimestamp: "2026-07-02T13:03:55Z" message: InferenceService [error-404-isvc-5b6c7] is Ready metadata: creationTimestamp: "2026-07-02T13:03:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:55Z" name: error-404-isvc-5b6c7.18be7ab872b67680 namespace: kserve-ci-e2e-test resourceVersion: "22693" uid: 7ee6ff96-416b-4d8b-b997-37dddb92de28 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:11:33.099150Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26231" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-63f60-predictor-58dc685fdb-6f6zf to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b231571f059 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: 940f59a3-0080-409c-b45b-db46a4630657 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-63f60-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b23215daf38 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: 3d49283d-7232-4351-94e4-69dcbc4fca99 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26251" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b2352f86cce namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 1d902634-f4d2-4885-9de5-0c1aa5d7f21f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b235480fa8f namespace: kserve-ci-e2e-test resourceVersion: "26281" uid: 5c609600-ecee-4569-8b78-50a47add48df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b235742490d namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 4eded6cb-5658-44db-97ef-8f0734107725 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b2357b5683c namespace: kserve-ci-e2e-test resourceVersion: "26287" uid: d20eed2e-2a3a-4614-a1df-71536cd8def6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b2357be0dd6 namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: 1cbc7e1e-dab5-44f4-bad5-76329f6724c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b235a672122 namespace: kserve-ci-e2e-test resourceVersion: "26291" uid: a6b7156e-eb66-4dce-aa2d-92b0d34adb24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b235ae0e6f5 namespace: kserve-ci-e2e-test resourceVersion: "26293" uid: e1dc64d0-6133-481f-816d-3ec693f2d2a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:12:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:10Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b2383c21e8c namespace: kserve-ci-e2e-test resourceVersion: "26590" uid: 1a363241-b86a-4050-9d48-e208cc5f977c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b33741de4c5 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: 03a3b6f0-ab81-4879-93ab-362ace716778 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b33741e7895 namespace: kserve-ci-e2e-test resourceVersion: "27085" uid: f13959f9-04da-4479-af62-9fe0b84ed839 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: b6d34e5f-d9f1-4a6e-889f-90ea3304f93d kind: Event lastTimestamp: "2026-07-02T13:12:45Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:45Z" name: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf.18be7b340bebb143 namespace: kserve-ci-e2e-test resourceVersion: "27197" uid: 404ccb5d-4406-4dc5-a6d5-12c5e0d39a49 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-63f60-predictor-58dc685fdb namespace: kserve-ci-e2e-test resourceVersion: "26214" uid: 45daa0ee-cd6f-4506-bcf0-6dda55613616 kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: 'Created pod: error-404-isvc-63f60-predictor-58dc685fdb-6f6zf' metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-63f60-predictor-58dc685fdb.18be7b2314ba1368 namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 055b5533-5340-4311-b515-a2c8efc05461 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: be9908ed-3599-4eff-a3d4-138bd00760f1 kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: Scaled up replica set error-404-isvc-63f60-predictor-58dc685fdb from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-63f60-predictor.18be7b231231a70d namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: c6153123-bee2-4a58-bd90-c65d61b78222 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:11:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 1fd76726-9082-4a99-ba21-a70114932a29 kind: Event lastTimestamp: "2026-07-02T13:12:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:03Z" name: error-404-isvc-63f60-predictor.18be7b2690b92d73 namespace: kserve-ci-e2e-test resourceVersion: "26536" uid: 9ef7042d-5900-429d-8dfc-3454fba9b7c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:11:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 1fd76726-9082-4a99-ba21-a70114932a29 kind: Event lastTimestamp: "2026-07-02T13:12:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:11:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:03Z" name: error-404-isvc-63f60-predictor.18be7b26910508b0 namespace: kserve-ci-e2e-test resourceVersion: "26538" uid: e73d0513-d83f-401f-924a-045e37a8bb0d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 1fd76726-9082-4a99-ba21-a70114932a29 kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:33Z" name: error-404-isvc-63f60-predictor.18be7b2d8e690145 namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: c8d5ab0d-5aef-4a6e-853b-20bf4534a833 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26419" uid: 1fd76726-9082-4a99-ba21-a70114932a29 kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:33Z" name: error-404-isvc-63f60-predictor.18be7b2d8ec9aef5 namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: 9ee8e442-db36-4adf-8a64-35f7ce804740 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 3d7de3ec-2213-40d2-92ff-c434d40024f5 kind: Event lastTimestamp: "2026-07-02T13:12:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:21Z" name: error-404-isvc-63f60.18be7b2312533ef0 namespace: kserve-ci-e2e-test resourceVersion: "26683" uid: 08b5ec5a-b273-46cc-96a8-2892ce678c0d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: 3d7de3ec-2213-40d2-92ff-c434d40024f5 kind: Event lastTimestamp: "2026-07-02T13:12:21Z" message: InferenceService [error-404-isvc-63f60] is Ready metadata: creationTimestamp: "2026-07-02T13:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:21Z" name: error-404-isvc-63f60.18be7b2e3f765f92 namespace: kserve-ci-e2e-test resourceVersion: "26678" uid: b09df9ae-308c-4b97-b4af-999daba4bb36 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:50:12.902531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:50:12Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79f903c7c943 namespace: kserve-ci-e2e-test resourceVersion: "15189" uid: ff0bdca4-a582-4c77-abfd-65e8e347dbce reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15194" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Add eth0 [10.134.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:50:13Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79f91d3a45e8 namespace: kserve-ci-e2e-test resourceVersion: "15241" uid: 96fcc683-837d-4245-98d2-92fb18d773ff reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79f91e80db1f namespace: kserve-ci-e2e-test resourceVersion: "15244" uid: bd528c4e-df23-41ff-afa9-919e9cedd552 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" in 15.923s (15.923s including waiting). Image size: 1334629471 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fcd39e9d74 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 27a2163d-1b99-4eb2-b959-a7d4b8c06a33 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fcd7a5d5dc namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 1c6c029c-7028-4ac7-84bc-43911115f7ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fcd81a5364 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: be7c5f60-9b5c-43e1-b151-b3e3f2b1f799 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fcd8254a26 namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: 8f3f4130-c0ee-499b-a934-9cff5a53ab8c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.521s (2.521s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fd6e6d0aa4 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 88132cd1-4a70-4c74-9178-fb3f54b928cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fd7252da06 namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: c6a921dc-1c90-407b-9602-0118de51ab0a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fd72d44496 namespace: kserve-ci-e2e-test resourceVersion: "15455" uid: 7ba7aaec-e4a6-4abc-838d-2ebf414973f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:51:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:08Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be79fd75c70c3f namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: 1b3ab637-a6fa-4b73-b3db-b18b0c2ce1ec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be7a0dfbd60c9d namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 9ad0c395-ae2f-422d-9c5b-f6a43f5533dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be7a0dfbd686c5 namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 642c1437-3f10-4833-9ace-8506b4ff77d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 7812dc6f-e3a1-42af-ba09-068b9ceed016 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Readiness probe failed: Get "https://10.134.0.20:8643/healthz": dial tcp 10.134.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw.18be7a0dfde8b07d namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 1f8c9d84-f26a-46ab-8f31-3a8166afa2b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-75b89-predictor-7f5dc87c4 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 3941e274-d3e0-4918-9a9f-6c674c08ed82 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'Created pod: error-404-isvc-75b89-predictor-7f5dc87c4-z2qpw' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: error-404-isvc-75b89-predictor-7f5dc87c4.18be79f9035afeec namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: d324681c-80de-41d0-8cc4-22986e4f5ade reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: 23daba63-659f-4d9b-8102-393d36e66259 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: Scaled up replica set error-404-isvc-75b89-predictor-7f5dc87c4 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: error-404-isvc-75b89-predictor.18be79f901ba9396 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 8bf3e1ab-873e-42ce-a6a8-192a9d8cdd48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 7def8cd7-3545-4876-b03b-139a91848bb0 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:57Z" name: error-404-isvc-75b89-predictor.18be79fc804a7bc1 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 1694f952-f0c6-48fe-9bdb-ba5fecaef692 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 7def8cd7-3545-4876-b03b-139a91848bb0 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:57Z" name: error-404-isvc-75b89-predictor.18be79fc80c5aad0 namespace: kserve-ci-e2e-test resourceVersion: "15706" uid: ebc8b8fb-4616-4a70-9310-d7c3a57b2d57 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 7def8cd7-3545-4876-b03b-139a91848bb0 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-75b89-predictor.18be7a06fd0a9118 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 75c33d75-e1e8-4a22-af3e-0556227c247d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 7def8cd7-3545-4876-b03b-139a91848bb0 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: error-404-isvc-75b89-predictor.18be7a06fd7471c0 namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 2d7a8b6f-531f-4a6a-a01d-b76fe7898228 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b89 namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 7e22ac57-5c66-451f-b55d-652f06ef86d7 kind: Event lastTimestamp: "2026-07-02T12:51:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:18Z" name: error-404-isvc-75b89.18be79f901bd4a13 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 41d5c6c6-9e7c-4688-b4ab-8f28d2d7eb4e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75b89 namespace: kserve-ci-e2e-test resourceVersion: "15886" uid: 7e22ac57-5c66-451f-b55d-652f06ef86d7 kind: Event lastTimestamp: "2026-07-02T12:51:18Z" message: InferenceService [error-404-isvc-75b89] is Ready metadata: creationTimestamp: "2026-07-02T12:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:18Z" name: error-404-isvc-75b89.18be7a0831035aec namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: c12167e4-748d-4a0b-8830-02cc1d26185e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:01:47.821060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7a990-predictor-9f9c44d9b-9v57k to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9ad0271655 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: bfe55157-bc2a-4476-97eb-b60b6524b7ca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7a990-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9ad79f186e namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: e552b5bf-b4ed-4f38-9334-da8a535436bd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:01:48Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b0d767f7f namespace: kserve-ci-e2e-test resourceVersion: "21214" uid: 97a89824-9f25-46e3-99d3-419eeedbe11b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b0fe6d936 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: 17bbdb2e-8bd0-40a7-b2e6-1a73e7a5d439 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b137aac23 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 08252447-164d-419e-80dd-5921d8ebf332 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b13eb4042 namespace: kserve-ci-e2e-test resourceVersion: "21218" uid: c8bdef28-020e-4095-b45b-e311481af46f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b13f5428e namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 98e9affb-6e6e-4992-901e-4d4ff3d5b4aa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:49Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b1801c133 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 14e14e26-bad4-4926-9235-d6bcaf21066f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:01:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:49Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b1888a53d namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: ee8114b2-0907-4bf9-8ba5-dd87647d5961 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:02:36Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:36Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7a9b7836fde9 namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 2c33f4a9-cd06-4a05-8f88-ac460c269c13 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:07Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7aad74e2fceb namespace: kserve-ci-e2e-test resourceVersion: "22166" uid: 4ceb105c-f095-4684-bda0-8dd3b9c72cc1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 1413e900-a9c6-4606-bcfe-b60c3813e400 kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:07Z" name: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k.18be7aad74e3b974 namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: 1257572f-6eab-4134-a9ba-ecf7a9514a85 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7a990-predictor-9f9c44d9b namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 1a0a8745-c0a6-4cf5-bcdd-0e9f8ff74d57 kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: 'Created pod: error-404-isvc-7a990-predictor-9f9c44d9b-9v57k' metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990-predictor-9f9c44d9b.18be7a9acfa22ea3 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 407b4503-e6c7-48c2-b0ec-c5c6a9283369 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 12494675-be57-4318-aab5-2bb70d277d57 kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Scaled up replica set error-404-isvc-7a990-predictor-9f9c44d9b from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990-predictor.18be7a9acdc63ec6 namespace: kserve-ci-e2e-test resourceVersion: "21156" uid: 03e9503d-e600-4b1e-b687-59324346eecf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: af883058-ae4d-4135-93dc-aa6fbb697a9f kind: Event lastTimestamp: "2026-07-02T13:02:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:17Z" name: error-404-isvc-7a990-predictor.18be7a9e4c661f53 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 44c09d0f-661c-4c32-ab15-6fa038749c42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: af883058-ae4d-4135-93dc-aa6fbb697a9f kind: Event lastTimestamp: "2026-07-02T13:02:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:17Z" name: error-404-isvc-7a990-predictor.18be7a9e4cc06600 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: ffcc650f-4612-46e7-bb84-9a8487329103 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:02:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: af883058-ae4d-4135-93dc-aa6fbb697a9f kind: Event lastTimestamp: "2026-07-02T13:03:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:02Z" name: error-404-isvc-7a990-predictor.18be7aa54a8cd59c namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: b810baf4-d4f7-42fa-98df-c6a9535704b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:02:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: af883058-ae4d-4135-93dc-aa6fbb697a9f kind: Event lastTimestamp: "2026-07-02T13:03:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:02Z" name: error-404-isvc-7a990-predictor.18be7aa54b1c8034 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 46855ce5-0443-4ca9-a9e2-660b0f6b1c0f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: fbac2f9b-8e07-4472-88ca-41bcb3971f5c kind: Event lastTimestamp: "2026-07-02T13:02:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:46Z" name: error-404-isvc-7a990.18be7a9acdd7fe08 namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: badc4427-f38b-46ad-8960-cd63ad553fd8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: fbac2f9b-8e07-4472-88ca-41bcb3971f5c kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-7a990": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7a990": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990.18be7a9ace4c7c94 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 63d91086-adae-42af-986c-dbbce2699e81 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21137" uid: fbac2f9b-8e07-4472-88ca-41bcb3971f5c kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7a990": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-7a990.18be7a9ace4cd02d namespace: kserve-ci-e2e-test resourceVersion: "21160" uid: ab8809cb-4bb3-4ffb-9e29-530a9afbab38 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21901" uid: fbac2f9b-8e07-4472-88ca-41bcb3971f5c kind: Event lastTimestamp: "2026-07-02T13:02:46Z" message: InferenceService [error-404-isvc-7a990] is Ready metadata: creationTimestamp: "2026-07-02T13:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:46Z" name: error-404-isvc-7a990.18be7aa88757b9a0 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: 01d220ad-4f81-4a88-80bf-ea57d364c48c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:52:22.850707Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16760" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8bebb-predictor-5796bd8b66-hvggm to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a17454b9aee namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 4ce24911-ec33-4879-9599-b3f9276f78fe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8bebb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a174cd8e7f8 namespace: kserve-ci-e2e-test resourceVersion: "16792" uid: f3febad9-2f9a-4d10-b967-0133c06d7245 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:52:23Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a17835d06fd namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: 78703683-df17-4b42-81b6-9e1a31d7cf6f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a1784bc87f3 namespace: kserve-ci-e2e-test resourceVersion: "16817" uid: f7fab45c-f8c1-4311-8382-669afc887716 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a178811d9ae namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 09e5c1bd-779d-49c3-8580-87e94d78284e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a17889d5b9c namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 0c67313b-4567-4214-9a20-c9392e81aebd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a1788a77284 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 79a26ffe-7790-4dfe-8f88-90e9009b9620 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:24Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a178b33df04 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: 8a14417d-4237-4b95-978b-50dfe252e212 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:52:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:24Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a178bf9f046 namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 8c30a5d5-40f1-4d06-97a5-3076e4a3ac13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:52:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T12:53:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:52:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:01Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a17dfc0a3d7 namespace: kserve-ci-e2e-test resourceVersion: "17200" uid: 998f58a4-70b1-46df-98bb-11f3b523fd00 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a9ad5aa7e76 namespace: kserve-ci-e2e-test resourceVersion: "21186" uid: 45cf857f-3008-4414-85d6-6092434f820b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 95ade19c-8fea-4214-8d6a-7c5264e03c0e kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm.18be7a9ad5aafb2c namespace: kserve-ci-e2e-test resourceVersion: "21189" uid: 5a6c1455-0a97-44c0-b426-301f6655244a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8bebb-predictor-5796bd8b66 namespace: kserve-ci-e2e-test resourceVersion: "16751" uid: b89d1cbb-f325-432b-8081-c1c5da822af3 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'Created pod: error-404-isvc-8bebb-predictor-5796bd8b66-hvggm' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb-predictor-5796bd8b66.18be7a1744a03ff7 namespace: kserve-ci-e2e-test resourceVersion: "16761" uid: b6ce9652-c8ce-4ebe-9b4f-6f6ad8404447 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16748" uid: fb315d5b-adfc-4334-93ab-bafa1247112e kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Scaled up replica set error-404-isvc-8bebb-predictor-5796bd8b66 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb-predictor.18be7a1742ecda95 namespace: kserve-ci-e2e-test resourceVersion: "16753" uid: f00cba4a-831a-4e92-a440-5912a2364be2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: e7757359-cf84-4d90-8f39-b555f64fc0be kind: Event lastTimestamp: "2026-07-02T12:52:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:52Z" name: error-404-isvc-8bebb-predictor.18be7a1ac158dda6 namespace: kserve-ci-e2e-test resourceVersion: "17129" uid: d7031c3c-bed2-4014-95d4-e8baf29a9737 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16750" uid: e7757359-cf84-4d90-8f39-b555f64fc0be kind: Event lastTimestamp: "2026-07-02T12:52:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:52Z" name: error-404-isvc-8bebb-predictor.18be7a1ac1b8313d namespace: kserve-ci-e2e-test resourceVersion: "17130" uid: 928fb971-6c21-4f3e-b801-d6aa09ba9ea4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:53:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: e7757359-cf84-4d90-8f39-b555f64fc0be kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: error-404-isvc-8bebb-predictor.18be7a21bfde6db5 namespace: kserve-ci-e2e-test resourceVersion: "17517" uid: 700ef838-3dde-4192-9ce9-b573fb94481b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:53:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: e7757359-cf84-4d90-8f39-b555f64fc0be kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: error-404-isvc-8bebb-predictor.18be7a21c0a546dc namespace: kserve-ci-e2e-test resourceVersion: "17518" uid: 9c45f627-7d75-45c4-83ce-f3e0644c449f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: efdc1537-3ff6-4ee1-baa9-f4e1bcd99d5d kind: Event lastTimestamp: "2026-07-02T12:53:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:11Z" name: error-404-isvc-8bebb.18be7a1742e5d89c namespace: kserve-ci-e2e-test resourceVersion: "17288" uid: 6b2c4ffb-94d8-4e2c-a73f-f1b07e0314a7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: efdc1537-3ff6-4ee1-baa9-f4e1bcd99d5d kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'Failed to update status for InferenceService "error-404-isvc-8bebb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8bebb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb.18be7a17438817f9 namespace: kserve-ci-e2e-test resourceVersion: "16756" uid: 094acabf-cc5c-41f3-b2ae-a90511d131d3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "16726" uid: efdc1537-3ff6-4ee1-baa9-f4e1bcd99d5d kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8bebb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:22Z" name: error-404-isvc-8bebb.18be7a174388761d namespace: kserve-ci-e2e-test resourceVersion: "16757" uid: 14cfe301-f343-4121-9385-f51ddec0dc07 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17286" uid: efdc1537-3ff6-4ee1-baa9-f4e1bcd99d5d kind: Event lastTimestamp: "2026-07-02T12:53:11Z" message: InferenceService [error-404-isvc-8bebb] is Ready metadata: creationTimestamp: "2026-07-02T12:53:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:11Z" name: error-404-isvc-8bebb.18be7a229a3e9785 namespace: kserve-ci-e2e-test resourceVersion: "17287" uid: 1cd0d9d3-1be6-45a8-b2d6-3cfa7a6970bf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:12:43.718591Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-99760-predictor-7d86666cd-zpgsr to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b3386b12db1 namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: 3aa02ace-2c48-46a5-a6fb-072fc5921f3b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-99760-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b3391159ee4 namespace: kserve-ci-e2e-test resourceVersion: "27152" uid: 8db3d1bd-6344-461c-924f-6e43879c4885 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33c472eedb namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: 4b0ba4d9-888d-47ab-9ab8-d4d5d996ca67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33c63ce508 namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: f0f88f91-450e-4aab-809b-d4b09f389926 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33c9eccdbb namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: be494c83-590b-466c-a46b-b56343a23a89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33ca654e00 namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: aa55a4f6-1015-4433-8667-4ed5743985d8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33ca6f77fe namespace: kserve-ci-e2e-test resourceVersion: "27176" uid: f83c36b5-b30c-4124-ad25-7806c651b305 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33cd4e3283 namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: e1aed81e-0949-4cc5-a05f-340f0d5e6100 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33cdc08c9f namespace: kserve-ci-e2e-test resourceVersion: "27178" uid: f0088303-770f-4a16-892e-8e18700dd2b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T13:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:22:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:22:01Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7b33e2b4e7e8 namespace: kserve-ci-e2e-test resourceVersion: "31285" uid: 395178da-4cef-4d43-b9f6-48f6b4fa64d7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:21:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:21:58Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7bb4a6656536 namespace: kserve-ci-e2e-test resourceVersion: "31259" uid: e3ccd0f5-860a-4402-8eaf-15a581fd999b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:21:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:21:58Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7bb4a6699851 namespace: kserve-ci-e2e-test resourceVersion: "31260" uid: 0dc68318-84e0-4e93-b5e6-5a6e21226e99 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-99760-predictor-7d86666cd-zpgsr namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: f400839a-4d36-49d1-ba41-113472187487 kind: Event lastTimestamp: "2026-07-02T13:22:01Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:22:01Z" name: error-404-isvc-99760-predictor-7d86666cd-zpgsr.18be7bb5572e4f80 namespace: kserve-ci-e2e-test resourceVersion: "31284" uid: eb35ed22-73f4-46b0-8d23-5ee2bbe425ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-99760-predictor-7d86666cd namespace: kserve-ci-e2e-test resourceVersion: "27128" uid: 07ba3248-1382-4126-b0b7-ac16542670da kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'Created pod: error-404-isvc-99760-predictor-7d86666cd-zpgsr' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760-predictor-7d86666cd.18be7b3385e0bdcf namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: 09636ef4-ea50-419b-96a4-46243e9e1853 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: 17f5aea0-d9cd-4903-8d3b-c34fed4dfa41 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Scaled up replica set error-404-isvc-99760-predictor-7d86666cd from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760-predictor.18be7b3382f535b5 namespace: kserve-ci-e2e-test resourceVersion: "27131" uid: f6b3591f-11b7-4452-968f-ff7d941d753a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: 0e0214fa-1af1-437d-bcbc-11987dbd7f82 kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: error-404-isvc-99760-predictor.18be7b37010878c6 namespace: kserve-ci-e2e-test resourceVersion: "27895" uid: e4a87250-0234-42dd-b3bb-8e9a132346ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: 0e0214fa-1af1-437d-bcbc-11987dbd7f82 kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: error-404-isvc-99760-predictor.18be7b37017e5082 namespace: kserve-ci-e2e-test resourceVersion: "27896" uid: 35cfd03b-b43a-4ad5-af6c-b541c914d3f8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: 0e0214fa-1af1-437d-bcbc-11987dbd7f82 kind: Event lastTimestamp: "2026-07-02T13:13:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:13Z" name: error-404-isvc-99760-predictor.18be7b3a8060c117 namespace: kserve-ci-e2e-test resourceVersion: "27436" uid: 02411ccf-4603-43b2-8d21-4850b660becb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: 0e0214fa-1af1-437d-bcbc-11987dbd7f82 kind: Event lastTimestamp: "2026-07-02T13:13:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:13:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:13Z" name: error-404-isvc-99760-predictor.18be7b3a80d3c15a namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: 34e7735a-5593-4ce0-8b4e-1a76eab2f181 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 17ee27f7-5d3f-469b-87bc-b85996524e27 kind: Event lastTimestamp: "2026-07-02T13:13:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:41Z" name: error-404-isvc-99760.18be7b3382f3b59e namespace: kserve-ci-e2e-test resourceVersion: "27721" uid: cfa34d53-fe73-416f-8dde-eafc6a3470c1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 17ee27f7-5d3f-469b-87bc-b85996524e27 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-99760": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99760": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760.18be7b33839c5b40 namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: a24b1ca6-9a44-4c4b-a33b-e594e96e2545 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27096" uid: 17ee27f7-5d3f-469b-87bc-b85996524e27 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-99760": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:43Z" name: error-404-isvc-99760.18be7b33839ca8da namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 44f12b27-c589-4cb9-a5b7-61bc92819de4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27716" uid: 17ee27f7-5d3f-469b-87bc-b85996524e27 kind: Event lastTimestamp: "2026-07-02T13:13:41Z" message: InferenceService [error-404-isvc-99760] is Ready metadata: creationTimestamp: "2026-07-02T13:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:41Z" name: error-404-isvc-99760.18be7b40f50b7754 namespace: kserve-ci-e2e-test resourceVersion: "27718" uid: 7059046c-cdc9-4542-94d1-47ec487d8d6d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:02:08.217304Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9f8fdd053f namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 84d90045-cb22-4b4b-9377-b02123112d43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b2e72-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9f9b9d174c namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: 79de8e7c-6f5a-4372-b80e-c5a6b7c65e22 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fce752b9d namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: 94192ca5-b720-4e84-934c-757fc95b0a79 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fcfe0af05 namespace: kserve-ci-e2e-test resourceVersion: "21566" uid: 062fe407-9bdf-4586-8b55-647cdc69d892 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fd2d8856c namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 1ffe4ba0-e2a9-4fd8-b531-93a6d121adba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fd35662ad namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 736c37cd-df82-4d9a-959f-3e6736fe7bb3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fd3618215 namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 42d0db81-51c4-469d-ad48-b94acff21cdf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fd8af74bc namespace: kserve-ci-e2e-test resourceVersion: "21575" uid: b42cd4bb-a45b-4254-84d4-d3d9a632d068 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7a9fd9291823 namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: cf07c415-5c6c-4c61-88d2-1ba1d46d14f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:02:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:02:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:46Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7aa0258bf089 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 6ff1c129-dd24-4ece-b8e9-2ef678959386 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7b23195b874c namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 8e9a0d8c-8108-49f5-aa95-9231dedca12e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 4ff8e35a-c9be-462d-abce-3ae66c142bd1 kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978.18be7b23195cdc45 namespace: kserve-ci-e2e-test resourceVersion: "26258" uid: 183fc08e-5564-496b-bd50-7ae76304071c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b2e72-predictor-59d9b5c96d namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 6039015a-0ec1-429f-b44f-bd182bffc15e kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'Created pod: error-404-isvc-b2e72-predictor-59d9b5c96d-xl978' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72-predictor-59d9b5c96d.18be7a9f8f667d77 namespace: kserve-ci-e2e-test resourceVersion: "21526" uid: 7009108d-6b9a-444a-83b7-b5d34e306d18 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: 33ac6226-d30d-4e89-905c-1635846d0e42 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: Scaled up replica set error-404-isvc-b2e72-predictor-59d9b5c96d from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72-predictor.18be7a9f8d79103f namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: bf4a9317-35eb-4d02-befa-286af1bfb49d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 6c468d67-84af-4422-b0eb-c92133f6bf53 kind: Event lastTimestamp: "2026-07-02T13:02:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:38Z" name: error-404-isvc-b2e72-predictor.18be7aa30c7a9ee3 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 1976185d-6604-41c5-bcd4-0096c8c82a7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 6c468d67-84af-4422-b0eb-c92133f6bf53 kind: Event lastTimestamp: "2026-07-02T13:02:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:38Z" name: error-404-isvc-b2e72-predictor.18be7aa30cf799d6 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 78487f42-0a7f-48f6-9bad-a2a4969c8b63 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:02:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 6c468d67-84af-4422-b0eb-c92133f6bf53 kind: Event lastTimestamp: "2026-07-02T13:03:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:23Z" name: error-404-isvc-b2e72-predictor.18be7aaa0b028f34 namespace: kserve-ci-e2e-test resourceVersion: "22433" uid: bbb3cc52-7428-4700-b49d-97dc4e7d7e63 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:02:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21704" uid: 6c468d67-84af-4422-b0eb-c92133f6bf53 kind: Event lastTimestamp: "2026-07-02T13:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:23Z" name: error-404-isvc-b2e72-predictor.18be7aaa0bd0014c namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: 83399a36-5cbf-4d23-9e06-7bd5b3218def reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 521bd51b-ae88-4020-8488-c256fc404bd4 kind: Event lastTimestamp: "2026-07-02T13:02:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:56Z" name: error-404-isvc-b2e72.18be7a9f8d672e2f namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: e5d1b6a9-2e62-4715-98bd-21e15b2136ee reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 521bd51b-ae88-4020-8488-c256fc404bd4 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-b2e72": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e72": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72.18be7a9f8e37947d namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: d61b77bb-b781-480e-be8d-c3836368a28b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 521bd51b-ae88-4020-8488-c256fc404bd4 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b2e72": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:08Z" name: error-404-isvc-b2e72.18be7a9f8e37eeae namespace: kserve-ci-e2e-test resourceVersion: "21521" uid: ea94f937-96e1-4b4b-b1e7-904dc69beecd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: 521bd51b-ae88-4020-8488-c256fc404bd4 kind: Event lastTimestamp: "2026-07-02T13:02:56Z" message: InferenceService [error-404-isvc-b2e72] is Ready metadata: creationTimestamp: "2026-07-02T13:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:56Z" name: error-404-isvc-b2e72.18be7aaae13fb681 namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 2b119d72-9d91-4271-b995-969491ddca53 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:00:57.877439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:00:57Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f2f47e1a7 namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 8be92246-ca7c-4bc6-a2f2-c30fac833517 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20641" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f49c304c2 namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 3007823b-e48c-4c8b-8079-a1da8b17923a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f4b1f4f11 namespace: kserve-ci-e2e-test resourceVersion: "20665" uid: 60894ac4-bc24-4464-9f73-6b4c6e965d16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f4df9fdb5 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 5b9e5eb5-1915-4126-bf09-0b2621ec6224 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f4e798b66 namespace: kserve-ci-e2e-test resourceVersion: "20673" uid: 0ae1749b-a829-458a-bddf-622f834160cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f4e82ad27 namespace: kserve-ci-e2e-test resourceVersion: "20675" uid: e77b1387-3403-4a32-8eaa-3d0714bce2e5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f52e5b840 namespace: kserve-ci-e2e-test resourceVersion: "20678" uid: 8d5f9e8e-fa46-439c-915a-03023f426250 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f53588df6 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: 9281167f-53a6-4dce-9d3e-876588732d29 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:01:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:35Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a8f8a602f80 namespace: kserve-ci-e2e-test resourceVersion: "20973" uid: f22afd3d-effb-4fc2-96c8-07fc1857a600 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:07Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a9f7e364f20 namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: d490f947-0dad-4901-a69e-cb5b2bcf2590 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:07Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7a9f7e372390 namespace: kserve-ci-e2e-test resourceVersion: "21478" uid: 8ef57bf6-a5f0-4b03-a68f-2b450d1cc86c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 5fea819a-f3de-4ac8-92fe-acc5a8227857 kind: Event lastTimestamp: "2026-07-02T13:02:10Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:02:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:10Z" name: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk.18be7aa012851822 namespace: kserve-ci-e2e-test resourceVersion: "21589" uid: f1772ff3-0c41-45f9-bf72-f40bda061bc5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c0346-predictor-6fd846fb8d namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: c14142a7-3130-4d23-8168-9afd2f45a9c7 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: 'Created pod: error-404-isvc-c0346-predictor-6fd846fb8d-bd5rk' metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:57Z" name: error-404-isvc-c0346-predictor-6fd846fb8d.18be7a8f2e8d2423 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 42b6b538-c731-423b-964c-ab0267dd60db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20611" uid: e2b2570a-835a-443c-9816-6949c9013cb2 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Scaled up replica set error-404-isvc-c0346-predictor-6fd846fb8d from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:57Z" name: error-404-isvc-c0346-predictor.18be7a8f2ca6ac69 namespace: kserve-ci-e2e-test resourceVersion: "20614" uid: 74f0f3c7-45ce-45d2-acb4-68bde58182d7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 566d571a-00b8-4d2a-ba75-1f1d30368d09 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: error-404-isvc-c0346-predictor.18be7a92ab17cd94 namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 6c5c1688-32c6-4209-95cd-4cc7cc67a2a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: 566d571a-00b8-4d2a-ba75-1f1d30368d09 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: error-404-isvc-c0346-predictor.18be7a92ab7d0fe8 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: cfeea3fd-8cd1-45fb-848b-00e7867af705 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "20591" uid: 119a993b-6dcd-4dae-b175-7e7ce7dba768 kind: Event lastTimestamp: "2026-07-02T13:01:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:45Z" name: error-404-isvc-c0346.18be7a8f2ca16910 namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: 20cf5f43-a4af-4209-814f-878cc865114c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "21066" uid: 119a993b-6dcd-4dae-b175-7e7ce7dba768 kind: Event lastTimestamp: "2026-07-02T13:01:45Z" message: InferenceService [error-404-isvc-c0346] is Ready metadata: creationTimestamp: "2026-07-02T13:01:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:45Z" name: error-404-isvc-c0346.18be7a9a45aa5b9f namespace: kserve-ci-e2e-test resourceVersion: "21067" uid: 2e0e8037-3ca8-41fb-83eb-809c3d418b9d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:12:23.058627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26770" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7674-predictor-79cc69864-jh7vj to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2eb7432207 namespace: kserve-ci-e2e-test resourceVersion: "26779" uid: 0ca30385-46d9-4db0-b156-41e755fa008f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f7674-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2ec08793be namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: a029c585-ce4c-4986-9b31-d4ced4930e2a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2ef4d8a29d namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 6ca9f075-672f-4812-888f-c161419c2157 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2ef6fa89d7 namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: 547b2b03-a411-4f1a-8653-ced4ed7ed2cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2efab0a438 namespace: kserve-ci-e2e-test resourceVersion: "26830" uid: 94884d99-bafa-490a-a27f-b49d093a9eec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2efb22ef33 namespace: kserve-ci-e2e-test resourceVersion: "26831" uid: 274e2540-31ae-4cf1-9ee8-64f6ceea06a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2efb2c9c69 namespace: kserve-ci-e2e-test resourceVersion: "26832" uid: 9bbb515f-5b3d-4cfa-bd57-13590f54e707 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2efe41fe1a namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 3202fba1-2d98-404e-bad4-732fa41ab6b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:12:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:24Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2efeb968e1 namespace: kserve-ci-e2e-test resourceVersion: "26835" uid: 58f9be80-b2ba-4818-82e3-40d2715292a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:12:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:13:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:12Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7b2f70c83e6c namespace: kserve-ci-e2e-test resourceVersion: "27425" uid: 94eb15bb-ee98-4e65-b889-2e834e96ffc0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:29:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:52Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7c23088a1d22 namespace: kserve-ci-e2e-test resourceVersion: "34536" uid: 15a0ccf7-66f1-4510-a5a6-956fb0e3bead reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7674-predictor-79cc69864-jh7vj namespace: kserve-ci-e2e-test resourceVersion: "26774" uid: d7d3331c-2c7d-4880-ab5a-aa63b2107a19 kind: Event lastTimestamp: "2026-07-02T13:29:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:52Z" name: error-404-isvc-f7674-predictor-79cc69864-jh7vj.18be7c23088a9a72 namespace: kserve-ci-e2e-test resourceVersion: "34537" uid: 84a9f533-7049-46ee-922b-071c3de7e24e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7674-predictor-79cc69864 namespace: kserve-ci-e2e-test resourceVersion: "26761" uid: 62330be3-fc19-4bb6-9417-c8a04827c08c kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'Created pod: error-404-isvc-f7674-predictor-79cc69864-jh7vj' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674-predictor-79cc69864.18be7b2eb626568b namespace: kserve-ci-e2e-test resourceVersion: "26775" uid: e1ff631a-1fc4-4074-9185-ea60ac8911ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26759" uid: 971d3246-4f09-4687-800f-b809ae6f05fa kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Scaled up replica set error-404-isvc-f7674-predictor-79cc69864 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674-predictor.18be7b2eb406bff5 namespace: kserve-ci-e2e-test resourceVersion: "26763" uid: 21d0f575-86e3-4b5b-8c04-6316782645fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 7e433ceb-7fc7-484e-b4f5-9d4dfeadd0ed kind: Event lastTimestamp: "2026-07-02T13:12:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:53Z" name: error-404-isvc-f7674-predictor.18be7b323303cd56 namespace: kserve-ci-e2e-test resourceVersion: "27264" uid: bca5a666-f8b9-4030-b076-97cc0d16c302 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26762" uid: 7e433ceb-7fc7-484e-b4f5-9d4dfeadd0ed kind: Event lastTimestamp: "2026-07-02T13:12:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:53Z" name: error-404-isvc-f7674-predictor.18be7b323428be86 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: f7c09f1c-0ca1-4f60-b0bf-e792955d3abf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:13:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 7e433ceb-7fc7-484e-b4f5-9d4dfeadd0ed kind: Event lastTimestamp: "2026-07-02T13:13:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:53Z" name: error-404-isvc-f7674-predictor.18be7b393286a78e namespace: kserve-ci-e2e-test resourceVersion: "27853" uid: 7fcb4fe4-dc5c-459a-ad39-9b3a43fdc8b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:13:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26994" uid: 7e433ceb-7fc7-484e-b4f5-9d4dfeadd0ed kind: Event lastTimestamp: "2026-07-02T13:13:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:53Z" name: error-404-isvc-f7674-predictor.18be7b3932eb63de namespace: kserve-ci-e2e-test resourceVersion: "27854" uid: 4a601a68-e649-48e5-95fd-789789c78a34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: f64a6e83-2a43-463a-8fe5-f5dc92cd3caa kind: Event lastTimestamp: "2026-07-02T13:13:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:22Z" name: error-404-isvc-f7674.18be7b2eb4671c20 namespace: kserve-ci-e2e-test resourceVersion: "27524" uid: e4b91626-ad46-4c41-93b2-9c933e1f7c40 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: f64a6e83-2a43-463a-8fe5-f5dc92cd3caa kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7674": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7674": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674.18be7b2eb4d4fc71 namespace: kserve-ci-e2e-test resourceVersion: "26768" uid: 824d2a66-97e1-401e-b4ba-f1d56f290e31 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: f64a6e83-2a43-463a-8fe5-f5dc92cd3caa kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7674": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:23Z" name: error-404-isvc-f7674.18be7b2eb4d539fe namespace: kserve-ci-e2e-test resourceVersion: "26769" uid: 313a9e30-2c3f-41d4-8f8f-db50035af717 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27521" uid: f64a6e83-2a43-463a-8fe5-f5dc92cd3caa kind: Event lastTimestamp: "2026-07-02T13:13:22Z" message: InferenceService [error-404-isvc-f7674] is Ready metadata: creationTimestamp: "2026-07-02T13:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:22Z" name: error-404-isvc-f7674.18be7b3c82d41870 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: b7dc3351-a5f0-4584-b0a2-96b6496c44be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:50:12.615036Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:50:12Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f8f2a50647 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 7e6c0fe0-b65d-4216-a7b4-24048f5f96af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:12Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f8fd44c140 namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: 48912d18-55d1-46f9-90d3-ae52509f6ef1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f92f8fa154 namespace: kserve-ci-e2e-test resourceVersion: "15250" uid: a50bb886-b583-4926-88d8-3a6e1ab6ae91 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f930da3582 namespace: kserve-ci-e2e-test resourceVersion: "15252" uid: abb9c776-2af7-4134-a3a5-c2daa8ac89b3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f9334f3346 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 8e9cf1a7-aed3-40e7-8b44-48d9e57ccda3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79f933bfa54c namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: ef619a48-8a3b-4f57-856f-163c254935b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:18Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" metadata: creationTimestamp: "2026-07-02T12:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:18Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79fa47a3b350 namespace: kserve-ci-e2e-test resourceVersion: "15320" uid: dca4d6be-b97d-47ff-8c50-91b60150a6aa reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" in 23.548s (23.548s including waiting). Image size: 1558018474 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:41Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79ffc33e0c36 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: af53d7cf-9ce8-44ff-a873-b39a45025adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:41Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79ffc7e056f7 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: ff2e98ee-df96-421c-a43b-a3bbe930d543 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:41Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79ffc906ba18 namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: a338ec30-0ff5-4f4e-a2b3-452c9461c39a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:50:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:41Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be79ffc913b2ab namespace: kserve-ci-e2e-test resourceVersion: "15550" uid: 5688d214-8795-4743-88da-cb85b2733c56 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.893s (2.893s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:44Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a00758b2a33 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: 026fa306-604e-4ed7-b5b4-88376e28003e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:44Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a007b7eeb7c namespace: kserve-ci-e2e-test resourceVersion: "15593" uid: ad70ea57-fb3b-4ea1-9c40-8ce9c4dce385 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:50:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:44Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a007cb0c90e namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 3f7d373e-36f1-4dfc-afdc-4babf0bc8e1a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T12:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:51:41Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:41Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a009963c948 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 47e85224-3306-4121-b0db-006317e2e429 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a173f341089 namespace: kserve-ci-e2e-test resourceVersion: "16740" uid: 16f65f11-0556-4fd6-940f-b280e25ec67d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a173f34a882 namespace: kserve-ci-e2e-test resourceVersion: "16741" uid: a9c487bc-eab4-4c4e-8010-391b65dbe2c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: 21f5a1ae-7b3c-468f-95c7-a90dcb90c4c5 kind: Event lastTimestamp: "2026-07-02T12:52:26Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:26Z" name: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg.18be7a181ddc585d namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 9c72ca8d-b42f-4d49-9a0e-e1aebbb2fec3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-786f6957cf namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 45aee393-14ff-46a9-98f4-920241048d17 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-786f6957cf-wv9rg' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: isvc-sklearn-graph-1-predictor-786f6957cf.18be79f8f2318438 namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: 7183ee0a-92df-4f5a-a1da-d98666d197e2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 705a3580-66a7-4c81-baae-c1c87754cf96 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-786f6957cf from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: isvc-sklearn-graph-1-predictor.18be79f8e9e25110 namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 73481008-5268-4244-984e-27addc08377c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: e17bab89-fe9b-4e1f-9406-71b4cc57f50c kind: Event lastTimestamp: "2026-07-02T12:51:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:12Z" name: isvc-sklearn-graph-1-predictor.18be79fc70dfb851 namespace: kserve-ci-e2e-test resourceVersion: "15819" uid: bf44779e-b10e-4b47-8f4b-4d4f0fd3e70f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15100" uid: e17bab89-fe9b-4e1f-9406-71b4cc57f50c kind: Event lastTimestamp: "2026-07-02T12:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:12Z" name: isvc-sklearn-graph-1-predictor.18be79fc71736dbd namespace: kserve-ci-e2e-test resourceVersion: "15820" uid: cd54513b-1e0b-41c8-a677-8a448c62c2bf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: e17bab89-fe9b-4e1f-9406-71b4cc57f50c kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: isvc-sklearn-graph-1-predictor.18be7a0371a16803 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 3a30f5a3-1fb8-4b3b-94c3-bdba2e87ed67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: e17bab89-fe9b-4e1f-9406-71b4cc57f50c kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: isvc-sklearn-graph-1-predictor.18be7a037212762c namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: fc1fc356-ea52-4884-9dc3-160d0087a028 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15084" uid: 1e4c50aa-2606-426e-803f-c7acc91801d9 kind: Event lastTimestamp: "2026-07-02T12:51:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:51Z" name: isvc-sklearn-graph-1.18be79f8eff2075f namespace: kserve-ci-e2e-test resourceVersion: "16366" uid: 97a83a47-f9a3-46f6-9df8-0323ce6bc621 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16359" uid: 1e4c50aa-2606-426e-803f-c7acc91801d9 kind: Event lastTimestamp: "2026-07-02T12:51:51Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-02T12:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:51Z" name: isvc-sklearn-graph-1.18be7a0ffd72dff4 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 03641f91-305b-43db-9834-25ffb4135136 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:50:13.013250Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd to ip-10-0-140-7.ec2.internal metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f90a614c40 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 7f402d29-6c95-4c0a-9d72-dc2bbbde4aba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f924594d95 namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: afc9d839-672d-46c3-8fdb-73f0595ddaf1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f925fcaf54 namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: eeb32af1-92d6-4cae-b390-2bc9c4a2e0a8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:16Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:16Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" in 3.462s (3.462s including waiting). Image size: 301878719 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:16Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f9f45f1550 namespace: kserve-ci-e2e-test resourceVersion: "15297" uid: 73e32f25-a856-4f92-b093-3f68d9052bbd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:17Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:17Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f9f81c8103 namespace: kserve-ci-e2e-test resourceVersion: "15299" uid: 9a40a257-9aa5-4fe1-89cf-ce30f0e11090 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:17Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:17Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:50:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:17Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79f9f88660ee namespace: kserve-ci-e2e-test resourceVersion: "15300" uid: 087248b3-b7ae-499f-aeb9-5518487183e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:20Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" metadata: creationTimestamp: "2026-07-02T12:50:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:20Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fad9efe837 namespace: kserve-ci-e2e-test resourceVersion: "15346" uid: 9dd5a00e-df9c-4b75-b4d3-852e73b00768 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:34Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1694" in 13.446s (13.446s including waiting). Image size: 1558018474 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:34Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fdfb6c4dc2 namespace: kserve-ci-e2e-test resourceVersion: "15484" uid: 531b6113-801f-4f83-a0af-1d1714d52eaf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:34Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fdff11eab8 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: b6057617-e6c1-469c-89e6-0041faaf1fd8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:34Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fdff809340 namespace: kserve-ci-e2e-test resourceVersion: "15486" uid: e477463d-aa95-4ee7-b16b-007df9301d3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:34Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:50:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:34Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fdff91a8b0 namespace: kserve-ci-e2e-test resourceVersion: "15487" uid: ea7231ef-f3f8-4332-9456-81b11879f1f1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:36Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.163s (2.163s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:36Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fe808a0b0d namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: eb093c20-fbbb-4eef-b9eb-924898545053 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:36Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fe83c6385f namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: d14c3d58-5d3a-4408-ac73-c0a0df0a3c35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:50:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:36Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fe842f1374 namespace: kserve-ci-e2e-test resourceVersion: "15504" uid: 71205cbe-a138-4b2f-93b2-cbc53433ff56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:50:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be79fe9a2ad3bb namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 255829f7-15ae-4b5a-ac42-4e981824f94e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be7a173bb1df41 namespace: kserve-ci-e2e-test resourceVersion: "16708" uid: fc248ba9-2bc1-4082-a307-493252e5bbd5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be7a173bb2efb2 namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 258fbedb-0df2-41af-899a-8d7bf762bced reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: eda568ff-555c-4dec-b550-7b68621524b4 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd.18be7a17489b75a4 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: 90f43e2d-4148-469d-9086-a5b043386102 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-6774b6c5fc namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: bb048ed8-7fd3-49de-bfdf-2767835eb8e9 kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-6774b6c5fc-9h4kd' metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:13Z" name: isvc-sklearn-graph-2-predictor-6774b6c5fc.18be79f909e3ba66 namespace: kserve-ci-e2e-test resourceVersion: "15211" uid: 0f09876d-cb7b-40c2-b973-1bac0739cacb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 01fe7fb0-2f23-463d-8c51-9c76f7586574 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-6774b6c5fc from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: isvc-sklearn-graph-2-predictor.18be79f907433127 namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 1055b540-94da-4f49-b45b-d89fe20eb2fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: e97de2d0-bd6e-4d36-b2dc-49fca9e7bac4 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:57Z" name: isvc-sklearn-graph-2-predictor.18be79fc85ceb37b namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: f2513145-dc6b-4747-9adb-aeee1b9b4d11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: e97de2d0-bd6e-4d36-b2dc-49fca9e7bac4 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:58Z" name: isvc-sklearn-graph-2-predictor.18be79fc863b3106 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 9de743df-eee1-4bf4-9b36-bb0486c666a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:51:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: e97de2d0-bd6e-4d36-b2dc-49fca9e7bac4 kind: Event lastTimestamp: "2026-07-02T12:52:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:13Z" name: isvc-sklearn-graph-2-predictor.18be7a070256a606 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 6bbbf28a-4c8c-46d9-8e0e-82bf93f26816 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:51:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: e97de2d0-bd6e-4d36-b2dc-49fca9e7bac4 kind: Event lastTimestamp: "2026-07-02T12:52:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:13Z" name: isvc-sklearn-graph-2-predictor.18be7a0702bd3f68 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 7c643685-895b-4c35-9306-829f093d86f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: b601ed39-069e-47d0-a3bb-95be222371fe kind: Event lastTimestamp: "2026-07-02T12:51:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:51Z" name: isvc-sklearn-graph-2.18be79f907436af3 namespace: kserve-ci-e2e-test resourceVersion: "16370" uid: 4248300d-32ec-4318-b538-68a80362e845 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16368" uid: b601ed39-069e-47d0-a3bb-95be222371fe kind: Event lastTimestamp: "2026-07-02T12:51:51Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-02T12:51:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:51Z" name: isvc-sklearn-graph-2.18be7a10046c46b9 namespace: kserve-ci-e2e-test resourceVersion: "16369" uid: e4a4905c-40f7-49e8-9196-fb68a328e6dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:50:12.835428Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-jt9qq to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:50:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f8ffc7ebbc namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 4e73a9bf-6d37-40b5-a977-cfca260a6675 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f90949337f namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: 6ac113f0-9039-4ebf-8aff-1a8f9d73a75e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:50:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f93e2d1639 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 41c37995-a84e-4a5d-a913-9572d0b3b84c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:ad817336055c6da6bf66cacbd06b857caf8adb146e60422365f2b38a3444569b" already present on machine metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f93f83ca29 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: e4cfeda1-2070-4f58-8cfa-443d7731e6a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f943c11238 namespace: kserve-ci-e2e-test resourceVersion: "15265" uid: bcfbe5a2-3a22-49d8-965b-b58947bcd0e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79f9442f67c8 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 4149c43a-7911-4a2d-9cee-6e5f88e3ef43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:18Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-02T12:50:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:18Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79fa47bc6a0d namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: b657cf6b-c337-4568-ad1f-a11465aaf7ac reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:42Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.968s (23.968s including waiting). Image size: 1305293048 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79ffdc5d72db namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: e96ead1a-d8dc-430f-bd46-ff26ca9c305e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79ffe1e7183f namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 47558966-7000-4d36-80cb-3c2de9243bb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79ffe2aefe23 namespace: kserve-ci-e2e-test resourceVersion: "15556" uid: b7081029-d3f9-4339-8683-f6782a96029e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:42Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be79ffe2baaf7c namespace: kserve-ci-e2e-test resourceVersion: "15557" uid: a89d6ed8-0dff-4d38-a52d-6afa873b0791 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.476s (2.476s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a00765d4c7d namespace: kserve-ci-e2e-test resourceVersion: "15592" uid: 88e3ec6e-4209-4e95-b861-25059689755f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a007cb1aef9 namespace: kserve-ci-e2e-test resourceVersion: "15595" uid: 18dc2d44-f800-47f6-8de0-665039e03ca4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:50:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:45Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a007e402f7a namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 5a3630bf-b936-4ace-a212-397211f10038 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T12:50:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:51:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:32Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a00d569c4a1 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 4d09e1b7-578e-4748-8e83-fe66750c4b9a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a1748faa409 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: 1fef5157-2783-4a75-8b9f-799e05658307 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq namespace: kserve-ci-e2e-test resourceVersion: "15163" uid: 69f33241-5f5d-4f71-918f-5dfa102fd0fb kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: isvc-xgboost-graph-predictor-555fc69d56-jt9qq.18be7a1748fb024e namespace: kserve-ci-e2e-test resourceVersion: "16789" uid: 9bcf9376-ce04-4998-86bf-85514620e0d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: af5340f4-8477-462c-8c69-7e52a25ac0ce kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-jt9qq' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: isvc-xgboost-graph-predictor-555fc69d56.18be79f8ff22f3f3 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: fbe2c993-6ccb-4faa-b030-f1cb9d2f5776 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 9bdc2d06-8253-4659-9fce-00ae8b82ccae kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: isvc-xgboost-graph-predictor.18be79f8fc3594d5 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 5485f134-50a5-488a-979e-5602d80fb456 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: d1359494-c13e-405e-93a3-9c15f9ef4b5d kind: Event lastTimestamp: "2026-07-02T12:51:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:12Z" name: isvc-xgboost-graph-predictor.18be79fc7b16ff61 namespace: kserve-ci-e2e-test resourceVersion: "15827" uid: 82231e68-0460-4aeb-80e8-8de1bd7acf91 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15149" uid: d1359494-c13e-405e-93a3-9c15f9ef4b5d kind: Event lastTimestamp: "2026-07-02T12:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:12Z" name: isvc-xgboost-graph-predictor.18be79fc7be16ce2 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 1f6d9e5b-64ba-463c-aca8-7e1cfeb5080a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: d1359494-c13e-405e-93a3-9c15f9ef4b5d kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: isvc-xgboost-graph-predictor.18be7a0379425e91 namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: fa332a79-b49a-4ab3-8679-e30d4f45fba8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15409" uid: d1359494-c13e-405e-93a3-9c15f9ef4b5d kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: isvc-xgboost-graph-predictor.18be7a0379aaa921 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: 1c2614b2-1e82-4b67-8659-a67bda28de0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 07d233c9-402a-4118-9722-fb7733ea39f2 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:42Z" name: isvc-xgboost-graph.18be79f8fc358acb namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 62b02654-1605-4af1-ad8f-fba6ff4d4f21 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16125" uid: 07d233c9-402a-4118-9722-fb7733ea39f2 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:42Z" name: isvc-xgboost-graph.18be7a0de5522b4d namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 7b879ef1-de68-482b-93df-dc02c05d71da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:52:12.465297Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-786fb4fbc7-6glcm to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:52:12Z" name: model-chainer-786fb4fbc7-6glcm.18be7a14da46e052 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: e50480ca-58e8-4a81-8346-266e67c9b433 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:12Z" name: model-chainer-786fb4fbc7-6glcm.18be7a14e77e5319 namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: 48078287-5533-4b09-81e2-9053f8189c83 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:13Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:13Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:52:13Z" name: model-chainer-786fb4fbc7-6glcm.18be7a1516ca871b namespace: kserve-ci-e2e-test resourceVersion: "16587" uid: b5dfeb01-66d7-4c20-acaf-e453022fd327 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:13Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" metadata: creationTimestamp: "2026-07-02T12:52:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:13Z" name: model-chainer-786fb4fbc7-6glcm.18be7a1518092bd8 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 3ab87790-e7e8-4024-94e5-bcab419c68b8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:15Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" in 2.008s (2.008s including waiting). Image size: 218070116 bytes.' metadata: creationTimestamp: "2026-07-02T12:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:15Z" name: model-chainer-786fb4fbc7-6glcm.18be7a158fc36308 namespace: kserve-ci-e2e-test resourceVersion: "16604" uid: cc9346a4-4501-448e-96b5-15bfca93999c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:15Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-02T12:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:15Z" name: model-chainer-786fb4fbc7-6glcm.18be7a159260e098 namespace: kserve-ci-e2e-test resourceVersion: "16605" uid: b942b361-507f-438b-ae0f-c84727deaea6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:15Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-02T12:52:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:15Z" name: model-chainer-786fb4fbc7-6glcm.18be7a1592bf0d28 namespace: kserve-ci-e2e-test resourceVersion: "16606" uid: c1ea142e-a064-40ba-bfc7-cf538263dfbc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:22Z" name: model-chainer-786fb4fbc7-6glcm.18be7a1734839198 namespace: kserve-ci-e2e-test resourceVersion: "16678" uid: 30f6f1ad-79da-4390-8ecc-d1f765811906 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:52:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-786fb4fbc7-6glcm namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8c57f64d-b239-4f40-b2cb-b204401c2570 kind: Event lastTimestamp: "2026-07-02T12:52:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:52:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:51Z" name: model-chainer-786fb4fbc7-6glcm.18be7a18312f8aec namespace: kserve-ci-e2e-test resourceVersion: "17116" uid: 1f52a11f-8229-4eff-a6e6-01b3ced2d9be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-786fb4fbc7 namespace: kserve-ci-e2e-test resourceVersion: "16545" uid: 8b268c6a-26fb-488d-8577-1ad64ef07e12 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'Created pod: model-chainer-786fb4fbc7-6glcm' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: model-chainer-786fb4fbc7.18be7a14d9ddea0b namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: 5a3001f8-259c-4467-86dc-2bba60896a8b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: ca3be527-121c-428f-a7ae-a77ac2c36393 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: Scaled up replica set model-chainer-786fb4fbc7 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: model-chainer.18be7a14d90a867c namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: ee3c9d0f-ae02-4c23-9794-f1ad6614fb5c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16560" uid: a14573f4-deb7-4e59-bedd-ba111b130d9b kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:12Z" name: model-chainer.18be7a14def2f6b7 namespace: kserve-ci-e2e-test resourceVersion: "16562" uid: 7d43d07e-5707-4f9a-8c2f-8cf27c98ffd5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:04:08.021125Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22804" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-5b6c7-7bcb5cf8c8-zvl47 to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:04:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:04:08Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abb74ba5088 namespace: kserve-ci-e2e-test resourceVersion: "22809" uid: e076faf0-0033-44a8-bf91-64d617041b2b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:08Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:04:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-5b6c7-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:04:08Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abb7f6b4761 namespace: kserve-ci-e2e-test resourceVersion: "22822" uid: 82ac8a22-5733-4945-b996-21c8e2f3a34d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:09Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22812" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:04:09Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:04:09Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abbb1b84d56 namespace: kserve-ci-e2e-test resourceVersion: "22833" uid: e108f390-ac33-4960-a224-509afc4cdc1e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b6c7} kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:04:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:04:09Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abbb327595e namespace: kserve-ci-e2e-test resourceVersion: "22835" uid: 889b389d-b79b-4d3c-8a6e-dca2faa60694 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b6c7} kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:04:09Z" message: 'Created container: sequence-graph-5b6c7' metadata: creationTimestamp: "2026-07-02T13:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:04:09Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abbb6343b02 namespace: kserve-ci-e2e-test resourceVersion: "22838" uid: 1abca9fe-dc59-4607-a524-90c9fc4221ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b6c7} kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:04:09Z" message: Started container sequence-graph-5b6c7 metadata: creationTimestamp: "2026-07-02T13:04:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:04:09Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7abbb6a1a4e7 namespace: kserve-ci-e2e-test resourceVersion: "22839" uid: 2313b8ac-6f43-4264-abe0-5bd36875131e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b6c7} kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:12:22Z" message: Stopping container sequence-graph-5b6c7 metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:22Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7b2ea4125955 namespace: kserve-ci-e2e-test resourceVersion: "26707" uid: 161eb76f-1b84-45ea-a1bd-802caa3188f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:12:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-5b6c7} kind: Pod name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 6ec7311f-0da4-4296-babf-b00cb78cea17 kind: Event lastTimestamp: "2026-07-02T13:12:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:12:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:50Z" name: sequence-graph-5b6c7-7bcb5cf8c8-zvl47.18be7b2f41cda8f9 namespace: kserve-ci-e2e-test resourceVersion: "27238" uid: f86871e8-7a08-4e4d-a176-eb720828dd98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-5b6c7-7bcb5cf8c8 namespace: kserve-ci-e2e-test resourceVersion: "22800" uid: 1a450bd7-6fcb-47ca-a1a3-6dca5eb7a76f kind: Event lastTimestamp: "2026-07-02T13:04:08Z" message: 'Created pod: sequence-graph-5b6c7-7bcb5cf8c8-zvl47' metadata: creationTimestamp: "2026-07-02T13:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:08Z" name: sequence-graph-5b6c7-7bcb5cf8c8.18be7abb7440635c namespace: kserve-ci-e2e-test resourceVersion: "22805" uid: de82cd78-59b4-4552-9e25-b013e9a9ea76 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22796" uid: 09d51637-2604-416c-9526-07a3317ef157 kind: Event lastTimestamp: "2026-07-02T13:04:07Z" message: Scaled up replica set sequence-graph-5b6c7-7bcb5cf8c8 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:07Z" name: sequence-graph-5b6c7.18be7abb734e7162 namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 128443f4-98a4-475c-b667-739ed07dd4e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:04:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22815" uid: fcf280e0-c037-49ed-ade5-8d89b4696377 kind: Event lastTimestamp: "2026-07-02T13:04:08Z" message: InferenceGraph [sequence-graph-5b6c7] is Ready metadata: creationTimestamp: "2026-07-02T13:04:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:04:08Z" name: sequence-graph-5b6c7.18be7abb78c4a607 namespace: kserve-ci-e2e-test resourceVersion: "22819" uid: 8ef89767-757a-499f-a724-1764d4eea8ee reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22799" uid: 6f1c983b-abb2-4f01-b90d-a160c2b07c6e kind: Event lastTimestamp: "2026-07-02T13:04:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:38Z" name: sequence-graph-5b6c7.18be7abef21b0db0 namespace: kserve-ci-e2e-test resourceVersion: "23052" uid: d4d8b8aa-ad1c-4b05-bf71-d0e490effe67 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22799" uid: 6f1c983b-abb2-4f01-b90d-a160c2b07c6e kind: Event lastTimestamp: "2026-07-02T13:04:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:04:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:38Z" name: sequence-graph-5b6c7.18be7abef295e764 namespace: kserve-ci-e2e-test resourceVersion: "23053" uid: b8b764d9-b8c7-4025-8909-a9284ec329f2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:02:57.708277Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22013" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7a990-795b58dd54-kbzhg to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:02:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:02:57Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab15c14eb4 namespace: kserve-ci-e2e-test resourceVersion: "22018" uid: 6f524b79-ab6e-41f3-8d20-44457cfac8a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:02:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7a990-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:57Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab1edb3f47 namespace: kserve-ci-e2e-test resourceVersion: "22030" uid: 192e86af-b8d7-406c-bb79-12a8be87afbf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:58Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22022" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:02:58Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:02:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:02:58Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab52d641f4 namespace: kserve-ci-e2e-test resourceVersion: "22040" uid: 4fda8972-a46c-4c0e-8bde-4646ef0cb65f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7a990} kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:02:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:58Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab5407e2b2 namespace: kserve-ci-e2e-test resourceVersion: "22042" uid: c6970946-931c-41ed-8812-2a7aef6ded38 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7a990} kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:02:58Z" message: 'Created container: sequence-graph-7a990' metadata: creationTimestamp: "2026-07-02T13:02:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:58Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab56c78e95 namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: e9596996-a5f5-484e-8310-36e069d9850a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7a990} kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:02:58Z" message: Started container sequence-graph-7a990 metadata: creationTimestamp: "2026-07-02T13:02:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:58Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aab573aa809 namespace: kserve-ci-e2e-test resourceVersion: "22046" uid: da3be404-c466-4f72-bd1e-7d63343ff191 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7a990} kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: Stopping container sequence-graph-7a990 metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:07Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aad6ee7080f namespace: kserve-ci-e2e-test resourceVersion: "22139" uid: 19b142f3-654c-4ea0-8596-9d3877aa4fc0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:03:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7a990} kind: Pod name: sequence-graph-7a990-795b58dd54-kbzhg namespace: kserve-ci-e2e-test resourceVersion: "22016" uid: ba9f521e-a2c7-43b2-8f10-88d6fa89d21e kind: Event lastTimestamp: "2026-07-02T13:03:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:35Z" name: sequence-graph-7a990-795b58dd54-kbzhg.18be7aadf9b97c47 namespace: kserve-ci-e2e-test resourceVersion: "22526" uid: 239a3b3d-802b-4860-9f90-1ebb066c1286 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7a990-795b58dd54 namespace: kserve-ci-e2e-test resourceVersion: "22010" uid: 9da8ac17-c0f9-48a4-a730-75c468dee58b kind: Event lastTimestamp: "2026-07-02T13:02:57Z" message: 'Created pod: sequence-graph-7a990-795b58dd54-kbzhg' metadata: creationTimestamp: "2026-07-02T13:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:57Z" name: sequence-graph-7a990-795b58dd54.18be7aab1546136b namespace: kserve-ci-e2e-test resourceVersion: "22015" uid: a6ffd69d-dce3-4bc7-b247-ea81af0717bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7a990 namespace: kserve-ci-e2e-test resourceVersion: "22008" uid: a2111164-d553-4da5-a411-3819f14ca70f kind: Event lastTimestamp: "2026-07-02T13:02:57Z" message: Scaled up replica set sequence-graph-7a990-795b58dd54 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:57Z" name: sequence-graph-7a990.18be7aab1460890c namespace: kserve-ci-e2e-test resourceVersion: "22012" uid: 2102dc49-466e-4f4e-8470-c4a67c921d9a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7a990 namespace: kserve-ci-e2e-test resourceVersion: "22025" uid: 805ad8c0-5209-4c9e-91db-1131fe5e6589 kind: Event lastTimestamp: "2026-07-02T13:02:57Z" message: InferenceGraph [sequence-graph-7a990] is Ready metadata: creationTimestamp: "2026-07-02T13:02:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:57Z" name: sequence-graph-7a990.18be7aab19ef0c7e namespace: kserve-ci-e2e-test resourceVersion: "22027" uid: 8b97b2c6-33a7-400e-a26d-76a1f3f16c0f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:53:22.793553Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17377" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8bebb-86485cbd97-g4qmq to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:53:22Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a253a2ad419 namespace: kserve-ci-e2e-test resourceVersion: "17385" uid: 9871f9cc-cf30-4554-a639-deb6c277edcd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17388" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T12:53:23Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:53:23Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a25530dc641 namespace: kserve-ci-e2e-test resourceVersion: "17402" uid: fe7eeab3-105d-4d99-82f1-06fe06372a96 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bebb} kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T12:53:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T12:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:23Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a25549f9d63 namespace: kserve-ci-e2e-test resourceVersion: "17405" uid: 163a6be1-76e0-4f3b-a876-500e4d502493 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bebb} kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T12:53:23Z" message: 'Created container: sequence-graph-8bebb' metadata: creationTimestamp: "2026-07-02T12:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:23Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a25572a1138 namespace: kserve-ci-e2e-test resourceVersion: "17406" uid: 1cd61960-62ae-46da-ac5e-81f9be0c6d34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bebb} kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T12:53:23Z" message: Started container sequence-graph-8bebb metadata: creationTimestamp: "2026-07-02T12:53:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:53:23Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a2557871650 namespace: kserve-ci-e2e-test resourceVersion: "17407" uid: 030792cc-942d-49cb-9641-3c2895caa1ba reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bebb} kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Stopping container sequence-graph-8bebb metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a9abe9b6f19 namespace: kserve-ci-e2e-test resourceVersion: "21097" uid: 940e6a9a-ae42-490b-89ff-d6842895352f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:01:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8bebb} kind: Pod name: sequence-graph-8bebb-86485cbd97-g4qmq namespace: kserve-ci-e2e-test resourceVersion: "17380" uid: 989e2bc5-363e-451f-8eb5-68fc40607d99 kind: Event lastTimestamp: "2026-07-02T13:02:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:01:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:15Z" name: sequence-graph-8bebb-86485cbd97-g4qmq.18be7a9b55bcc242 namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: 90d18d68-8d6a-42b8-82f5-df592af82c7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8bebb-86485cbd97 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 9f7af2d6-952e-41bf-85cc-162b6f20c281 kind: Event lastTimestamp: "2026-07-02T12:53:22Z" message: 'Created pod: sequence-graph-8bebb-86485cbd97-g4qmq' metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:22Z" name: sequence-graph-8bebb-86485cbd97.18be7a25394ed2ab namespace: kserve-ci-e2e-test resourceVersion: "17381" uid: af1bd708-7954-449e-8450-efd79160f56a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17372" uid: aaa8672a-6645-4db8-88fa-3acf15b64a21 kind: Event lastTimestamp: "2026-07-02T12:53:22Z" message: Scaled up replica set sequence-graph-8bebb-86485cbd97 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:22Z" name: sequence-graph-8bebb.18be7a253754335f namespace: kserve-ci-e2e-test resourceVersion: "17376" uid: a4a53d66-a100-423c-af3d-6707b35f6469 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17393" uid: 0ca2f31f-9b41-4669-92d1-2ad914bdc388 kind: Event lastTimestamp: "2026-07-02T12:53:22Z" message: InferenceGraph [sequence-graph-8bebb] is Ready metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:22Z" name: sequence-graph-8bebb.18be7a2540ed8bdd namespace: kserve-ci-e2e-test resourceVersion: "17396" uid: 8d68e9a4-fbbb-4d7b-be86-7234e51f5323 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 44a23728-6cbb-4e75-8162-a5fdee5b2b42 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: sequence-graph-8bebb.18be7a28b6802715 namespace: kserve-ci-e2e-test resourceVersion: "17511" uid: f96c8cf6-c1bd-4e17-b4d7-3e52e33159f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: 44a23728-6cbb-4e75-8162-a5fdee5b2b42 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:53:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: sequence-graph-8bebb.18be7a28b6cdaa65 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 6f45c29c-7dfa-4c49-bd4e-8dd9249bae88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 44a23728-6cbb-4e75-8162-a5fdee5b2b42 kind: Event lastTimestamp: "2026-07-02T12:53:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:52Z" name: sequence-graph-8bebb.18be7a2c36500f18 namespace: kserve-ci-e2e-test resourceVersion: "17623" uid: 1a15905e-d913-4e02-9498-db45f63341c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17512" uid: 44a23728-6cbb-4e75-8162-a5fdee5b2b42 kind: Event lastTimestamp: "2026-07-02T12:53:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:52Z" name: sequence-graph-8bebb.18be7a2c36d66b9e namespace: kserve-ci-e2e-test resourceVersion: "17624" uid: 9e14cc71-61ff-4f24-9bd5-eacae9eb9192 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:12:32.990236Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-63f60-76d54bfbbb-bdcgl to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:12:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:12:32Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b31073b6bf1 namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: 0cb6a1ae-842b-49e5-89ad-e9a53329960f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:33Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:12:33Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b312084cda1 namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 2c69a9cc-f225-4a75-afb9-968d07df84a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-63f60} kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:33Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b3121b7e63e namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 209531c2-1a17-469a-b8f9-c3d6c4570110 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-63f60} kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: 'Created container: splitter-graph-63f60' metadata: creationTimestamp: "2026-07-02T13:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:33Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b31245569bc namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 6bf1736b-224c-40ad-8b01-417bf3a8b95b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-63f60} kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: Started container splitter-graph-63f60 metadata: creationTimestamp: "2026-07-02T13:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:33Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b3124c448f4 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: 00bc64fd-239a-4df4-921c-0695e6796867 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-63f60} kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Stopping container splitter-graph-63f60 metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b3367b27c40 namespace: kserve-ci-e2e-test resourceVersion: "27057" uid: 9d9cba7d-2a81-44ce-b8df-f7ec0f4c01e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-63f60} kind: Pod name: splitter-graph-63f60-76d54bfbbb-bdcgl namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: a3d4ff50-0fb6-46f3-b3ac-eb78aff9536d kind: Event lastTimestamp: "2026-07-02T13:13:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:10Z" name: splitter-graph-63f60-76d54bfbbb-bdcgl.18be7b33ddb8159f namespace: kserve-ci-e2e-test resourceVersion: "27408" uid: 177e1c8f-f8e4-4134-bed5-1f0a3f5b28f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-63f60-76d54bfbbb namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: 3b0aa4c1-a012-4e61-b9b1-a76c622298ee kind: Event lastTimestamp: "2026-07-02T13:12:32Z" message: 'Created pod: splitter-graph-63f60-76d54bfbbb-bdcgl' metadata: creationTimestamp: "2026-07-02T13:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:32Z" name: splitter-graph-63f60-76d54bfbbb.18be7b3106c91eb8 namespace: kserve-ci-e2e-test resourceVersion: "26929" uid: 6d434e17-dce6-46e1-95ee-a0cd4ddac904 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: 9a4839bf-3d89-492a-b73a-a8c75bf3185e kind: Event lastTimestamp: "2026-07-02T13:12:32Z" message: Scaled up replica set splitter-graph-63f60-76d54bfbbb from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:12:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:32Z" name: splitter-graph-63f60.18be7b31061be5da namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: 8272a598-dba0-4005-841e-134c3e20e2c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 8ba88cb0-efcf-4d2a-8a0b-ce8b241bac0e kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: InferenceGraph [splitter-graph-63f60] is Ready metadata: creationTimestamp: "2026-07-02T13:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:33Z" name: splitter-graph-63f60.18be7b310b5828cd namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 4e5d683d-94d9-490c-b244-482989a4d19f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:13:43.479158Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27747" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-99760-7486b86459-ktkmd to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7b4170b2f368 namespace: kserve-ci-e2e-test resourceVersion: "27752" uid: e96e6efb-c812-40d1-9ff5-72fcb2928cad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27758" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7b4189cb2a6d namespace: kserve-ci-e2e-test resourceVersion: "27772" uid: 0be1189e-0545-4852-a851-6e11739a61c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-99760} kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7b418bea4673 namespace: kserve-ci-e2e-test resourceVersion: "27774" uid: 4edd1984-528a-45c6-81cd-be816a51c331 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-99760} kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: 'Created container: splitter-graph-99760' metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7b418e80ea94 namespace: kserve-ci-e2e-test resourceVersion: "27775" uid: d04aaf02-e523-4332-a7e2-63fc4d5c4ffc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-99760} kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: Started container splitter-graph-99760 metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7b418ef041a8 namespace: kserve-ci-e2e-test resourceVersion: "27776" uid: 98e85346-d054-4d0f-8877-07c87704e2e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-99760} kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:21:58Z" message: Stopping container splitter-graph-99760 metadata: creationTimestamp: "2026-07-02T13:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:21:58Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7bb496e14a0a namespace: kserve-ci-e2e-test resourceVersion: "31227" uid: d762f6e2-a190-4022-98b6-618b364058df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:22:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-99760} kind: Pod name: splitter-graph-99760-7486b86459-ktkmd namespace: kserve-ci-e2e-test resourceVersion: "27748" uid: f67dd30f-da2a-4489-bb0e-df1d9d8c0f60 kind: Event lastTimestamp: "2026-07-02T13:22:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:22:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:22:25Z" name: splitter-graph-99760-7486b86459-ktkmd.18be7bb526346dfb namespace: kserve-ci-e2e-test resourceVersion: "31462" uid: 1b875407-5f2d-4a6d-9324-9a78dc945bef reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-99760-7486b86459 namespace: kserve-ci-e2e-test resourceVersion: "27744" uid: b90586be-275a-435a-92e9-a04465cee1e7 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: 'Created pod: splitter-graph-99760-7486b86459-ktkmd' metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760-7486b86459.18be7b417011fbab namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: d382b689-c4a9-49d2-9a67-27759ad75b69 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-99760 namespace: kserve-ci-e2e-test resourceVersion: "27741" uid: b4535900-d3cb-4fb0-8fb1-cb2f604203bd kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: Scaled up replica set splitter-graph-99760-7486b86459 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760.18be7b416eef336b namespace: kserve-ci-e2e-test resourceVersion: "27746" uid: e3f7fdf4-46d1-4d8b-8b04-7345b2091276 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-99760 namespace: kserve-ci-e2e-test resourceVersion: "27765" uid: 3776448d-ec11-4e4a-b891-9cd93e091999 kind: Event lastTimestamp: "2026-07-02T13:13:43Z" message: InferenceGraph [splitter-graph-99760] is Ready metadata: creationTimestamp: "2026-07-02T13:13:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:43Z" name: splitter-graph-99760.18be7b4178d85b22 namespace: kserve-ci-e2e-test resourceVersion: "27767" uid: 7efb7eb2-523d-4797-9b2f-878bf56e14f0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-99760 namespace: kserve-ci-e2e-test resourceVersion: "27742" uid: 47d0bf89-adab-4196-94c2-3c8a1840355d kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: splitter-graph-99760.18be7b44ee616df6 namespace: kserve-ci-e2e-test resourceVersion: "27888" uid: a60d7613-6249-4808-a292-cac9b824d8c1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-99760 namespace: kserve-ci-e2e-test resourceVersion: "27742" uid: 47d0bf89-adab-4196-94c2-3c8a1840355d kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: splitter-graph-99760.18be7b44ef55f944 namespace: kserve-ci-e2e-test resourceVersion: "27889" uid: 5a86fff9-f033-447c-aeb4-0da6e0e12d05 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:51:42.813250Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16178" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-04e3b-predictor-7448945865-gfvcv to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0df2e0746f namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: ca229b4b-eaa2-4e9f-bd11-8985633969ab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16192" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e0c35556e namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 251a5b14-43ed-4919-bdc0-1e15efe5ad9f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e0d865454 namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 2476d231-44d6-4b12-8daf-0bff99371e95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e10e33696 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 468554ce-050f-4353-80a5-53b7fbf0ce38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e1171ba91 namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 386e3d3f-0c50-4885-8d07-88f8051de0b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e117dcd34 namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 4d201875-0718-46cc-8778-a8c5a7819ed9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e14489d6d namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 367ad8fc-4710-40a5-a306-d3f1c93aeb11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T12:51:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e14bc0851 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 2013524c-3891-4ba1-ab39-ba20aee99a28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T12:51:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T13:01:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:00Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a0e4a291ad7 namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 776fccb9-cec9-429e-b594-a9fe3d1c7eaa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a8f2963266c namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: 8891561f-0001-4c14-9e3a-f3d28357ad4c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a8f29644758 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: eb91ddaa-2d83-4838-8c9c-a06e518050d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-04e3b-predictor-7448945865-gfvcv namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 7ea27268-cfef-4c78-a7e0-7b6a57bc0ee3 kind: Event lastTimestamp: "2026-07-02T13:01:00Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:00Z" name: success-200-isvc-04e3b-predictor-7448945865-gfvcv.18be7a8fbea0f096 namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: 909ad9e8-25f9-4cfc-92fe-86670d629336 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-04e3b-predictor-7448945865 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: 21fb0dc0-e396-4b52-8343-c74a0bdd1692 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Created pod: success-200-isvc-04e3b-predictor-7448945865-gfvcv' metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-04e3b-predictor-7448945865.18be7a0df260f52d namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: 590e5647-7667-4ee0-9ae1-efcbb6871dd4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16154" uid: 1cf5a006-38b2-41b4-8790-d6418991b6d8 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Scaled up replica set success-200-isvc-04e3b-predictor-7448945865 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-04e3b-predictor.18be7a0df063698c namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 32a5c374-d7ef-4d8c-aa7c-1bcb9df5ae1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 6f802667-80e8-470d-b92a-54a83792b71c kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: success-200-isvc-04e3b-predictor.18be7a116f62147c namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: d7d8cc1d-5bd8-4fe2-8be2-9166a7aedef3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:51:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16156" uid: 6f802667-80e8-470d-b92a-54a83792b71c kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:51:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: success-200-isvc-04e3b-predictor.18be7a116fdf27e2 namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: e280dcbc-1436-4e0b-a573-8369c5d1370c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 6f802667-80e8-470d-b92a-54a83792b71c kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: success-200-isvc-04e3b-predictor.18be7a14efac38d6 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: 313baee6-4d22-4f63-8f4f-a55dea2036bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-04e3b-predictor namespace: kserve-ci-e2e-test resourceVersion: "16414" uid: 6f802667-80e8-470d-b92a-54a83792b71c kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:12Z" name: success-200-isvc-04e3b-predictor.18be7a14f0189139 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: fd23c18c-bfc6-4c4c-89fd-2190cbbd1170 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 5775d0a5-7c97-4841-9ed2-d61ab7fac753 kind: Event lastTimestamp: "2026-07-02T12:52:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:30Z" name: success-200-isvc-04e3b.18be7a0df065a373 namespace: kserve-ci-e2e-test resourceVersion: "16913" uid: 92daee20-ad61-41b9-8887-34faf908dbcb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-04e3b namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: 5775d0a5-7c97-4841-9ed2-d61ab7fac753 kind: Event lastTimestamp: "2026-07-02T12:52:30Z" message: InferenceService [success-200-isvc-04e3b] is Ready metadata: creationTimestamp: "2026-07-02T12:52:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:30Z" name: success-200-isvc-04e3b.18be7a1907ef09fe namespace: kserve-ci-e2e-test resourceVersion: "16911" uid: bb266bc4-4b01-4981-8569-2ae2df1d8042 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:03:07.946356Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:03:07Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aad77fe0efd namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: 12cd30b2-921e-4552-af86-6ac53cbe5312 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5b6c7-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aad7f9eb5af namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: fba6a046-980e-4b9d-9d8c-136ab6390522 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22186" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:03:08Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadb54f5445 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: 415397b9-8ee3-410f-8c68-92e2ed667947 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadb6b45ee5 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: 27016d84-6a17-415c-974f-3a4fda1b49cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadba962598 namespace: kserve-ci-e2e-test resourceVersion: "22269" uid: a11dbcf2-c2e1-486f-981e-0c6b7c13bb34 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadbb16de7a namespace: kserve-ci-e2e-test resourceVersion: "22270" uid: f1bf8a9e-de38-4e20-aa02-855b1c74b5fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadbb20ef8f namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: 6428c292-8eb1-4aea-ae8a-5fe2606369b4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadbefa65c2 namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: f6b92502-6540-4b0b-a984-d46b426d0b1e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:09Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aadbf691e63 namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: c5b91c54-7da3-4dcf-9c1a-f69c17ead9a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:03:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:03:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:47Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7aae2d7e5e10 namespace: kserve-ci-e2e-test resourceVersion: "22618" uid: 2a50378e-1d73-4d95-904f-b63e12339048 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:12:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:22Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7b2eaf0972e9 namespace: kserve-ci-e2e-test resourceVersion: "26739" uid: 2751bbf1-35a8-4852-bf2a-6075689fd3c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5 namespace: kserve-ci-e2e-test resourceVersion: "22178" uid: 59c38447-8a34-40df-9cbb-71e57dbb7e5a kind: Event lastTimestamp: "2026-07-02T13:12:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:22Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5.18be7b2eaf0a0e02 namespace: kserve-ci-e2e-test resourceVersion: "26745" uid: 8b82b43d-3dfe-4cac-b0f8-7822e31c1a62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5b6c7-predictor-6954f8fdff namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: e93aaac4-231c-4d9c-88cf-2c500b9a64f0 kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: 'Created pod: success-200-isvc-5b6c7-predictor-6954f8fdff-xx8t5' metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:07Z" name: success-200-isvc-5b6c7-predictor-6954f8fdff.18be7aad77b8d3a4 namespace: kserve-ci-e2e-test resourceVersion: "22179" uid: e6738fe6-aec6-4b3d-be24-87ea20433152 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: fe0a8826-0fbb-489d-82ae-955877234945 kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: Scaled up replica set success-200-isvc-5b6c7-predictor-6954f8fdff from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:07Z" name: success-200-isvc-5b6c7-predictor.18be7aad75e1567d namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: abc2f434-f871-4b85-83b4-e3aad3ae1dfc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:03:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 52a8134b-8830-4878-84ca-c2dcfdb63972 kind: Event lastTimestamp: "2026-07-02T13:03:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:52Z" name: success-200-isvc-5b6c7-predictor.18be7ab0f51877bb namespace: kserve-ci-e2e-test resourceVersion: "22662" uid: e633cedd-682f-4e55-ae4e-d393e4745f1f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:03:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: 52a8134b-8830-4878-84ca-c2dcfdb63972 kind: Event lastTimestamp: "2026-07-02T13:03:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:03:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:52Z" name: success-200-isvc-5b6c7-predictor.18be7ab0f58ada7b namespace: kserve-ci-e2e-test resourceVersion: "22664" uid: af9cb0d0-30aa-4975-b488-e55c55e13368 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 52a8134b-8830-4878-84ca-c2dcfdb63972 kind: Event lastTimestamp: "2026-07-02T13:04:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:23Z" name: success-200-isvc-5b6c7-predictor.18be7abb72838d72 namespace: kserve-ci-e2e-test resourceVersion: "22946" uid: efd501a5-9e35-48af-94e0-d7e5b20a20eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:04:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5b6c7-predictor namespace: kserve-ci-e2e-test resourceVersion: "22423" uid: 52a8134b-8830-4878-84ca-c2dcfdb63972 kind: Event lastTimestamp: "2026-07-02T13:04:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:04:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:04:23Z" name: success-200-isvc-5b6c7-predictor.18be7abb72f4bb15 namespace: kserve-ci-e2e-test resourceVersion: "22947" uid: 5bcaafb4-4c04-4d25-a6a8-0f667c143c32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22144" uid: 18ac7b2d-c59d-471d-b866-461e1148b7ac kind: Event lastTimestamp: "2026-07-02T13:03:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:57Z" name: success-200-isvc-5b6c7.18be7aad75dbca73 namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: 54605d2e-3792-4ba1-9931-e0b39aab64ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22144" uid: 18ac7b2d-c59d-471d-b866-461e1148b7ac kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-5b6c7": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5b6c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:07Z" name: success-200-isvc-5b6c7.18be7aad765bca84 namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 93006c5f-dc3c-4881-8d41-47c323819690 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22144" uid: 18ac7b2d-c59d-471d-b866-461e1148b7ac kind: Event lastTimestamp: "2026-07-02T13:03:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-5b6c7": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:03:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:07Z" name: success-200-isvc-5b6c7.18be7aad765c0b95 namespace: kserve-ci-e2e-test resourceVersion: "22175" uid: 02dc6a70-be32-4aa6-83ac-3648d1964baa reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5b6c7 namespace: kserve-ci-e2e-test resourceVersion: "22713" uid: 18ac7b2d-c59d-471d-b866-461e1148b7ac kind: Event lastTimestamp: "2026-07-02T13:03:57Z" message: InferenceService [success-200-isvc-5b6c7] is Ready metadata: creationTimestamp: "2026-07-02T13:03:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:57Z" name: success-200-isvc-5b6c7.18be7ab8e9accedd namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: 339a0ea7-b69c-40b3-97f3-5b4d7a8cec08 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:11:33.007230Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-63f60-predictor-fc76d56f5-rrcpx to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:11:33Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b230ff7a77c namespace: kserve-ci-e2e-test resourceVersion: "26202" uid: e650a5bc-10a5-4e39-990c-5e1aaa896b93 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-63f60-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b231b557a4a namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: b730b09d-1716-4736-91db-3aa36598043a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b234d9bcb60 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 8425d894-420c-455e-a6d4-cfc3d7924209 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b2352f5ca86 namespace: kserve-ci-e2e-test resourceVersion: "26278" uid: 33f655c5-5f5c-4a0c-89ef-73796920190d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b2356ffa5b7 namespace: kserve-ci-e2e-test resourceVersion: "26283" uid: ec94e08f-e7eb-4fde-b684-19d339f4db56 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b235773a249 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 22894598-891f-4c67-a530-274e56faa3c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b2357800371 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: 6b6f4b94-1f6d-4f10-a842-c96326580efb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b235a56cd89 namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: 03eefd40-ae94-47e5-8463-1307d2296a07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:11:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:34Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b235ad5e7a8 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 407189d1-0941-40a9-b227-3dc65adae5dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:11:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:12:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:11:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:10Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b2383aca5df namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 435aae17-24de-4a12-aad3-9f3d4b6e70b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b3381497341 namespace: kserve-ci-e2e-test resourceVersion: "27125" uid: f9504534-00cf-4e0c-8833-fb29cfe75c10 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b338149dfd5 namespace: kserve-ci-e2e-test resourceVersion: "27127" uid: 62b147fb-a1b2-4968-b91d-bbfdcbb18893 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 233babba-5bbc-4832-9ba6-06063f0edd75 kind: Event lastTimestamp: "2026-07-02T13:12:45Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:45Z" name: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx.18be7b340becbee2 namespace: kserve-ci-e2e-test resourceVersion: "27198" uid: 4444bd5f-be52-4611-8565-f920e7692c57 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-63f60-predictor-fc76d56f5 namespace: kserve-ci-e2e-test resourceVersion: "26186" uid: 351399cb-868b-450d-8877-da0de4edc74a kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: 'Created pod: success-200-isvc-63f60-predictor-fc76d56f5-rrcpx' metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:11:33Z" name: success-200-isvc-63f60-predictor-fc76d56f5.18be7b230f8b5b8f namespace: kserve-ci-e2e-test resourceVersion: "26199" uid: 15679fff-e8c7-4fde-b22d-ecff4bfbfc8a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26184" uid: 92f9109f-0b7f-4e66-8f35-b10828002697 kind: Event lastTimestamp: "2026-07-02T13:11:32Z" message: Scaled up replica set success-200-isvc-63f60-predictor-fc76d56f5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:11:32Z" name: success-200-isvc-63f60-predictor.18be7b230bf474d6 namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: a9dc1086-0c0f-4815-8c25-234972b8b9b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:11:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 0a4743e7-714e-4d58-8b79-0659b47e0b64 kind: Event lastTimestamp: "2026-07-02T13:12:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:02Z" name: success-200-isvc-63f60-predictor.18be7b268a6768e2 namespace: kserve-ci-e2e-test resourceVersion: "26530" uid: 22916e17-0d46-42bc-901a-39f0dbe11f71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:11:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 0a4743e7-714e-4d58-8b79-0659b47e0b64 kind: Event lastTimestamp: "2026-07-02T13:12:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:11:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:02Z" name: success-200-isvc-63f60-predictor.18be7b268accac44 namespace: kserve-ci-e2e-test resourceVersion: "26531" uid: ab8c3d74-7804-406f-b6da-e93af4bab48c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 0a4743e7-714e-4d58-8b79-0659b47e0b64 kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:33Z" name: success-200-isvc-63f60-predictor.18be7b2d8890858c namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 477cf427-fd3f-49b9-b4f5-7d5cdb3930a8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-63f60-predictor namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 0a4743e7-714e-4d58-8b79-0659b47e0b64 kind: Event lastTimestamp: "2026-07-02T13:12:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:12:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:33Z" name: success-200-isvc-63f60-predictor.18be7b2d8923d4b2 namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: 37386c49-37b9-4c64-8dc0-6ab8009a8bfc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:11:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 548e8135-fc1e-4afc-a0e2-20d0a3ea0989 kind: Event lastTimestamp: "2026-07-02T13:12:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:21Z" name: success-200-isvc-63f60.18be7b230bf7efdf namespace: kserve-ci-e2e-test resourceVersion: "26684" uid: 71bd06a9-a804-4aa2-b7d9-367c5ef37849 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 548e8135-fc1e-4afc-a0e2-20d0a3ea0989 kind: Event lastTimestamp: "2026-07-02T13:11:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-63f60": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-63f60": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:11:32Z" name: success-200-isvc-63f60.18be7b230c66ca07 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 2dafc566-491a-478d-8fce-93e47037fe6a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 548e8135-fc1e-4afc-a0e2-20d0a3ea0989 kind: Event lastTimestamp: "2026-07-02T13:11:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-63f60": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:11:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:11:32Z" name: success-200-isvc-63f60.18be7b230c671b03 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 0e2ad46c-6fe8-4e1e-b123-9636741bb686 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-63f60 namespace: kserve-ci-e2e-test resourceVersion: "26681" uid: 548e8135-fc1e-4afc-a0e2-20d0a3ea0989 kind: Event lastTimestamp: "2026-07-02T13:12:21Z" message: InferenceService [success-200-isvc-63f60] is Ready metadata: creationTimestamp: "2026-07-02T13:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:21Z" name: success-200-isvc-63f60.18be7b2e423d119e namespace: kserve-ci-e2e-test resourceVersion: "26682" uid: f2a9eb8b-e26c-419f-8184-6e533f113a86 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:50:12.703557Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:50:12Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79f8f7ebae71 namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 90ccc3e3-f1eb-4c87-ac55-4cf77d6bb60e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-75b89-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:12Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79f90131c384 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: 06742a9c-de17-4e4a-b7df-969448d5abcf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Add eth0 [10.134.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:50:13Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79f934f37e3f namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 09270eda-a5b6-4a4a-9d4a-553e6b1c3d53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:13Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" metadata: creationTimestamp: "2026-07-02T12:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:13Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79f93647a40d namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 666a02a5-4226-417b-a558-5a2ca9a4c0d9 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" in 15.522s (15.522s including waiting). Image size: 1335607397 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fcd37eaacf namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 8eaad357-8734-4e26-9956-10831f66008a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fcd79a51e9 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: 752424d6-802b-42e9-8a82-2fd9142831f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fcd80cdfef namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: eaedf050-08c3-413f-8fba-43ed4bf257ac reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:29Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-02T12:50:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:29Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fcd819006f namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 90cd4f37-d530-4a59-a880-e58b1c7f0717 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.545s (2.545s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fd6fd38229 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 5f6c05fb-c1e2-461b-a282-9e65c056507c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fd726802eb namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: cb41986e-6f43-43e1-a995-45a44c667a42 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:50:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:50:31Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fd7320cb0b namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: cacf4dee-4b70-4e1a-8f70-9186d052f43e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:50:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:51:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:50:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:08Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be79fd75b46dc3 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: 52c66c5c-f369-4b9b-b0fd-b53472d5e3e2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be7a0df26512c7 namespace: kserve-ci-e2e-test resourceVersion: "16182" uid: bf44981d-c54a-43d3-96c1-20ce709af2ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be7a0df266450d namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: d3298e48-487e-41f5-940a-41c14242332e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq namespace: kserve-ci-e2e-test resourceVersion: "15135" uid: e5e22202-c96f-4bdd-81e2-8ea4bed2503e kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'Readiness probe failed: Get "https://10.134.0.19:8643/healthz": dial tcp 10.134.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:51:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:43Z" name: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq.18be7a0dfde8de73 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: d2e8b3e9-9b1c-40fa-8ab7-88200948614f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-75b89-predictor-864c8fdb5f namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 12421c76-07f2-433c-9693-95b3114067bc kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: 'Created pod: success-200-isvc-75b89-predictor-864c8fdb5f-k6tdq' metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: success-200-isvc-75b89-predictor-864c8fdb5f.18be79f8f77803ea namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: b8cec888-21f5-4e82-a2bc-0755fcd715ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: 2a44e899-2660-4784-942a-c29948daac20 kind: Event lastTimestamp: "2026-07-02T12:50:12Z" message: Scaled up replica set success-200-isvc-75b89-predictor-864c8fdb5f from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:12Z" name: success-200-isvc-75b89-predictor.18be79f8f5dfc69a namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 17de5106-cbf4-48d9-b0bc-b4177694a76d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: a8f43a0a-06aa-4e03-a5e6-3bcdbe5e3008 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:57Z" name: success-200-isvc-75b89-predictor.18be79fc74b2f108 namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 0a480648-2b3b-4396-88ba-2f62e9b65f47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: a8f43a0a-06aa-4e03-a5e6-3bcdbe5e3008 kind: Event lastTimestamp: "2026-07-02T12:50:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:50:57Z" name: success-200-isvc-75b89-predictor.18be79fc7587c3d6 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 41009062-9b98-436b-9f5a-f446b9d9be78 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: a8f43a0a-06aa-4e03-a5e6-3bcdbe5e3008 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-75b89-predictor.18be79fff447cb53 namespace: kserve-ci-e2e-test resourceVersion: "16170" uid: 3d811cf6-552d-4d3c-8c04-c1f9dc8b2d16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T12:50:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75b89-predictor namespace: kserve-ci-e2e-test resourceVersion: "15406" uid: a8f43a0a-06aa-4e03-a5e6-3bcdbe5e3008 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:50:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:42Z" name: success-200-isvc-75b89-predictor.18be79fff48f38b7 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 8b89ef34-8b74-4462-9a11-2b770b3a8244 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b89 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 9f8bca44-05ad-4658-ac53-46519f99e151 kind: Event lastTimestamp: "2026-07-02T12:51:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:18Z" name: success-200-isvc-75b89.18be79f8f5e34431 namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: ea5faa13-8971-491e-bfa8-532b8698ffb8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75b89 namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 9f8bca44-05ad-4658-ac53-46519f99e151 kind: Event lastTimestamp: "2026-07-02T12:51:18Z" message: InferenceService [success-200-isvc-75b89] is Ready metadata: creationTimestamp: "2026-07-02T12:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:18Z" name: success-200-isvc-75b89.18be7a083347e25b namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: d4b30de4-bcdb-4616-b59e-ea0d98632df3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:01:47.709439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7a990-predictor-7b76c758b9-pbfdd to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:01:47Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ac97fcebb namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 47404a5d-fe1e-40a9-9895-9421c93dbb7f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21145" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ae3af2a7a namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: bccbc30d-ee9e-4d31-9522-1f1bc995196e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ae6041d55 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: f7176690-4ccf-4b3b-ae59-0304eb66fa77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ae8dcdf17 namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 522f1623-4847-4ee4-bdee-5d97ce28ceb8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ae9506ff3 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 1a392e05-965f-4fb2-8456-03a89e8a83e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9ae95a1a88 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: c8d47cb1-8a7d-43e5-9d88-2c84d2651f84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9aebeec0fd namespace: kserve-ci-e2e-test resourceVersion: "21204" uid: 07e8d2f8-e9f5-477e-9e0c-b99871038a88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:01:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:48Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9aec5f9f9c namespace: kserve-ci-e2e-test resourceVersion: "21205" uid: 16a42334-9d67-465a-9cb1-03446553b2de reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:02:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:35Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7a9b3c52b03d namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 55905bbb-0047-433c-ad0c-a9d12b47194e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7aad7d46ab80 namespace: kserve-ci-e2e-test resourceVersion: "22218" uid: 1b184153-a923-4d7e-88f0-52b225c11f3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:03:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:08Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7aad7d473d2f namespace: kserve-ci-e2e-test resourceVersion: "22224" uid: 8821ef66-6973-4ef3-ac61-d307b4d2e87f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: c4d57fc5-4f72-40e4-bbbc-fcff17489250 kind: Event lastTimestamp: "2026-07-02T13:03:10Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:03:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:03:10Z" name: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd.18be7aae1891513a namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 5d59c28e-e0f7-4325-b563-a6f17faf3ee5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7a990-predictor-7b76c758b9 namespace: kserve-ci-e2e-test resourceVersion: "21112" uid: 6e85e4da-fd01-42b7-b75a-8782ae31924b kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: 'Created pod: success-200-isvc-7a990-predictor-7b76c758b9-pbfdd' metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:47Z" name: success-200-isvc-7a990-predictor-7b76c758b9.18be7a9ac8f218ee namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: afc85cd7-62ee-40a7-b3d5-fc77b257fa4d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21109" uid: 058bf17a-f88c-4092-a70f-bfbd57bf7295 kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Scaled up replica set success-200-isvc-7a990-predictor-7b76c758b9 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:47Z" name: success-200-isvc-7a990-predictor.18be7a9ac65feb87 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 820e1e34-c80c-4a26-90d4-042828e10cde reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 860c69d1-909d-472d-a983-1b3e72a9a656 kind: Event lastTimestamp: "2026-07-02T13:02:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:02Z" name: success-200-isvc-7a990-predictor.18be7a9e44efeb5e namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 56cd9fcd-6c13-4e63-a210-2a172e67ef46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21111" uid: 860c69d1-909d-472d-a983-1b3e72a9a656 kind: Event lastTimestamp: "2026-07-02T13:02:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:02Z" name: success-200-isvc-7a990-predictor.18be7a9e45623883 namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: 8221cac4-4513-4bf5-928d-f98edf9bc542 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 860c69d1-909d-472d-a983-1b3e72a9a656 kind: Event lastTimestamp: "2026-07-02T13:03:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:02Z" name: success-200-isvc-7a990-predictor.18be7aa1c4124add namespace: kserve-ci-e2e-test resourceVersion: "22076" uid: a90cff64-0738-4add-8708-ec0a28c32bae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7a990-predictor namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: 860c69d1-909d-472d-a983-1b3e72a9a656 kind: Event lastTimestamp: "2026-07-02T13:03:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:02Z" name: success-200-isvc-7a990-predictor.18be7aa1c493acd3 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: a73a731e-b5c1-421b-9001-69b55e7b32d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 9394c54a-1c2f-4d48-8cd9-5253b36cb9ef kind: Event lastTimestamp: "2026-07-02T13:02:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:45Z" name: success-200-isvc-7a990.18be7a9ac66b6b37 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: 50938127-c73f-4276-82cf-44b94f7152b8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7a990 namespace: kserve-ci-e2e-test resourceVersion: "21887" uid: 9394c54a-1c2f-4d48-8cd9-5253b36cb9ef kind: Event lastTimestamp: "2026-07-02T13:02:45Z" message: InferenceService [success-200-isvc-7a990] is Ready metadata: creationTimestamp: "2026-07-02T13:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:45Z" name: success-200-isvc-7a990.18be7aa84b47739b namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: 801366d8-1fad-4289-8ec4-3ca761d47f1e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:52:22.726102Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16720" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8bebb-predictor-674985687f-rxfmm to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:52:22Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a173dde51a1 namespace: kserve-ci-e2e-test resourceVersion: "16729" uid: e82099eb-c223-4b15-91c9-d2de76058b63 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16739" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a1757ec2b89 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 704fdec8-fd49-4ad7-96d5-0aa0c02fc67c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a1759835c92 namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: e8460017-a7f1-4d3e-87db-fc5b1bf5e94c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a175c3ddecb namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 40e72579-44f5-4a95-a46a-76c43c1c79e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a175cb00d6a namespace: kserve-ci-e2e-test resourceVersion: "16804" uid: f9533353-55e5-4aeb-8caf-372a8203f4c5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a175cb9eb46 namespace: kserve-ci-e2e-test resourceVersion: "16805" uid: b8a7dfc5-bc87-4872-9832-d395f722ebbf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a175fa7653f namespace: kserve-ci-e2e-test resourceVersion: "16807" uid: 90d38d91-c873-4fdb-b754-f55dd6129f04 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T12:52:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:23Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a1760189781 namespace: kserve-ci-e2e-test resourceVersion: "16808" uid: 1a1348a6-87a8-4bb6-9b5e-9ee25feb0f6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T12:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T13:01:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T12:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:49Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a1767c61b68 namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 3ffe1c9d-2812-48c2-94db-e5eccfc52004 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a9ac88d694e namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: a6d630fa-76ef-4c59-be6f-e514c5577abe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T13:01:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:01:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:47Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a9ac88e016a namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: 1afeb08c-e88d-4474-ad27-bbdef17ca95a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8bebb-predictor-674985687f-rxfmm namespace: kserve-ci-e2e-test resourceVersion: "16723" uid: 689be393-9525-45ff-93b3-ba50eeaa6dde kind: Event lastTimestamp: "2026-07-02T13:01:49Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:01:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:49Z" name: success-200-isvc-8bebb-predictor-674985687f-rxfmm.18be7a9b307c53f2 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 110958a4-00cf-479b-8dde-9fc44989ad8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8bebb-predictor-674985687f namespace: kserve-ci-e2e-test resourceVersion: "16706" uid: a91f377c-8bef-41ef-a488-77b98a6d6500 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: 'Created pod: success-200-isvc-8bebb-predictor-674985687f-rxfmm' metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:22Z" name: success-200-isvc-8bebb-predictor-674985687f.18be7a173d62377e namespace: kserve-ci-e2e-test resourceVersion: "16722" uid: 98c12aa5-3a7f-4285-a2dc-b464eb79e628 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 0dcfa6dd-95d8-4ed0-abdf-663fa6c74854 kind: Event lastTimestamp: "2026-07-02T12:52:22Z" message: Scaled up replica set success-200-isvc-8bebb-predictor-674985687f from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:22Z" name: success-200-isvc-8bebb-predictor.18be7a173bbb74f0 namespace: kserve-ci-e2e-test resourceVersion: "16707" uid: 00075113-7a70-4dbb-8605-105863c2c058 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 4a8e3761-eaa4-4f08-85f0-bd5fb4f80384 kind: Event lastTimestamp: "2026-07-02T12:53:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:07Z" name: success-200-isvc-8bebb-predictor.18be7a1aba46bb30 namespace: kserve-ci-e2e-test resourceVersion: "17241" uid: dfe0fd28-9503-48a1-af82-530ca89d845b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T12:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 4a8e3761-eaa4-4f08-85f0-bd5fb4f80384 kind: Event lastTimestamp: "2026-07-02T12:53:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:07Z" name: success-200-isvc-8bebb-predictor.18be7a1abaed980e namespace: kserve-ci-e2e-test resourceVersion: "17242" uid: 6a4c6c48-30dd-4e61-9355-d9915a0c8805 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:53:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 4a8e3761-eaa4-4f08-85f0-bd5fb4f80384 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: success-200-isvc-8bebb-predictor.18be7a2539dcd077 namespace: kserve-ci-e2e-test resourceVersion: "17514" uid: 0e5e309f-108a-4dca-9817-0446762ac005 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:53:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8bebb-predictor namespace: kserve-ci-e2e-test resourceVersion: "16972" uid: 4a8e3761-eaa4-4f08-85f0-bd5fb4f80384 kind: Event lastTimestamp: "2026-07-02T12:53:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T12:53:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:37Z" name: success-200-isvc-8bebb-predictor.18be7a253a824b63 namespace: kserve-ci-e2e-test resourceVersion: "17515" uid: ece67580-1278-4f16-89d0-129dcc23dbd9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T12:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "16684" uid: 9cb42ce2-6824-4993-a8e0-86d8519c5bcb kind: Event lastTimestamp: "2026-07-02T12:53:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T12:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:09Z" name: success-200-isvc-8bebb.18be7a173bbd8521 namespace: kserve-ci-e2e-test resourceVersion: "17265" uid: 53dc8af0-081b-4d40-9a86-a8a8cf38eb0f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:53:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8bebb namespace: kserve-ci-e2e-test resourceVersion: "17263" uid: 9cb42ce2-6824-4993-a8e0-86d8519c5bcb kind: Event lastTimestamp: "2026-07-02T12:53:09Z" message: InferenceService [success-200-isvc-8bebb] is Ready metadata: creationTimestamp: "2026-07-02T12:53:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:53:09Z" name: success-200-isvc-8bebb.18be7a2223652783 namespace: kserve-ci-e2e-test resourceVersion: "17264" uid: 0b2ef743-3c54-44a8-b0e1-c4ecacdf5fc8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:12:43.525228Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27092" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-99760-predictor-7f5586f59f-lnjjn to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b337b2a98a7 namespace: kserve-ci-e2e-test resourceVersion: "27102" uid: 678cb2bf-e8a8-4c5f-80dc-c78e22c59db4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b3395974ea9 namespace: kserve-ci-e2e-test resourceVersion: "27154" uid: 565aafdd-9ef8-40d3-8a7e-61e1baa35b17 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b3397ea5056 namespace: kserve-ci-e2e-test resourceVersion: "27156" uid: 89f1ea41-427e-4aa3-a645-e5d9efc461a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b339abc9f3f namespace: kserve-ci-e2e-test resourceVersion: "27157" uid: b60a9383-0c89-4246-abde-8518007d57c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b339b2f788e namespace: kserve-ci-e2e-test resourceVersion: "27158" uid: b5032255-02dc-45b2-8f26-07918327b7f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b339b3940c7 namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: 277a1843-cc9c-4350-aab1-53e6669e579b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b339fcbadcf namespace: kserve-ci-e2e-test resourceVersion: "27160" uid: ef6252a1-69b7-460e-8a47-453c9a0cf47c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:12:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:44Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b33a0425d43 namespace: kserve-ci-e2e-test resourceVersion: "27161" uid: c047f055-6659-42ef-a464-9513ccc8270a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:22:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:22:01Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7b33e2b5fcd4 namespace: kserve-ci-e2e-test resourceVersion: "31286" uid: 4a598a19-b78d-4cba-a9ae-3ff33f67c03d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:21:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:21:58Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7bb4a2f5d472 namespace: kserve-ci-e2e-test resourceVersion: "31244" uid: 93af59bc-5887-4b8a-a3ec-427952edfcbe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:21:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:21:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:21:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:21:58Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7bb4a2f67621 namespace: kserve-ci-e2e-test resourceVersion: "31245" uid: 5b4a69ab-9f06-444b-8195-9ccc262a00d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:22:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn namespace: kserve-ci-e2e-test resourceVersion: "27094" uid: e9aa7af2-62fc-41ec-bb5d-9e50722eb2aa kind: Event lastTimestamp: "2026-07-02T13:22:01Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:22:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:22:01Z" name: success-200-isvc-99760-predictor-7f5586f59f-lnjjn.18be7bb5572d10a8 namespace: kserve-ci-e2e-test resourceVersion: "31283" uid: 795994c3-2035-4e23-b38a-dcf70d5b81a3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-99760-predictor-7f5586f59f namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: ad4d6e95-b0af-4183-abe3-cc0867d8611a kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'Created pod: success-200-isvc-99760-predictor-7f5586f59f-lnjjn' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760-predictor-7f5586f59f.18be7b3379e1139d namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: fa02e0d0-4e42-47d4-a39f-e8be7095c295 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27076" uid: 229c16d3-ef83-44e9-ba1e-6e75499a0b8b kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: Scaled up replica set success-200-isvc-99760-predictor-7f5586f59f from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760-predictor.18be7b3374f3dbc3 namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 28d81ea4-702d-4e40-99d8-42cb0365f30c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: b81c0a7f-9105-48df-936b-bbdeaf792c75 kind: Event lastTimestamp: "2026-07-02T13:13:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:13Z" name: success-200-isvc-99760-predictor.18be7b36f44578b9 namespace: kserve-ci-e2e-test resourceVersion: "27434" uid: baccbb93-d545-46d4-a4af-4384bb24776d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: b81c0a7f-9105-48df-936b-bbdeaf792c75 kind: Event lastTimestamp: "2026-07-02T13:13:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:13Z" name: success-200-isvc-99760-predictor.18be7b36f4ad9f09 namespace: kserve-ci-e2e-test resourceVersion: "27435" uid: 6ce6b9a3-f9e1-4734-b6cc-1521bc514705 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:13:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: b81c0a7f-9105-48df-936b-bbdeaf792c75 kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: success-200-isvc-99760-predictor.18be7b3df293bf66 namespace: kserve-ci-e2e-test resourceVersion: "27891" uid: 3e663d56-a66a-4e8a-9f62-fc26479e8e4c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:13:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-99760-predictor namespace: kserve-ci-e2e-test resourceVersion: "27314" uid: b81c0a7f-9105-48df-936b-bbdeaf792c75 kind: Event lastTimestamp: "2026-07-02T13:13:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:58Z" name: success-200-isvc-99760-predictor.18be7b3df33c426d namespace: kserve-ci-e2e-test resourceVersion: "27892" uid: 4d2c7d36-7640-46a7-a4f8-20298c5a5199 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 072f6826-4474-413e-8f3c-028d18c3db31 kind: Event lastTimestamp: "2026-07-02T13:13:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:31Z" name: success-200-isvc-99760.18be7b3375bbf130 namespace: kserve-ci-e2e-test resourceVersion: "27605" uid: d96f036c-ec09-4ebf-abf2-e35019c62b9b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 072f6826-4474-413e-8f3c-028d18c3db31 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-99760": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-99760": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760.18be7b3376ac9c08 namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 474c2097-d9d5-4e6e-8fda-fed24b0ad9c9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27060" uid: 072f6826-4474-413e-8f3c-028d18c3db31 kind: Event lastTimestamp: "2026-07-02T13:12:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-99760": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:12:43Z" name: success-200-isvc-99760.18be7b3376acdce9 namespace: kserve-ci-e2e-test resourceVersion: "27091" uid: 7bea121a-5d64-41c7-847a-36fccba5d044 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-99760 namespace: kserve-ci-e2e-test resourceVersion: "27599" uid: 072f6826-4474-413e-8f3c-028d18c3db31 kind: Event lastTimestamp: "2026-07-02T13:13:31Z" message: InferenceService [success-200-isvc-99760] is Ready metadata: creationTimestamp: "2026-07-02T13:13:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:31Z" name: success-200-isvc-99760.18be7b3ea3557c07 namespace: kserve-ci-e2e-test resourceVersion: "27604" uid: 0bbca892-eeb6-4bc4-a7e2-c93ef88264b3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:02:08.047447Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b2e72-predictor-6cbff7944f-jcljw to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:02:08Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9f85bd3053 namespace: kserve-ci-e2e-test resourceVersion: "21489" uid: d6de7e3b-489b-4303-9c95-5441f1b3860a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b2e72-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:08Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9f8f8f3d81 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: f19db272-12fb-4f83-940f-aad370ccbaf0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fc3003347 namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: d65732f5-1a04-465a-ae86-896e805fd85c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fc45cddf6 namespace: kserve-ci-e2e-test resourceVersion: "21558" uid: f5ce6573-8874-4edd-b12b-66d7f939c8a1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fc802ace7 namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: 1194fb54-5fc6-4906-bbc1-7ce6efe0101c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fc87c8538 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: 969f2f47-9eeb-45e2-9483-201ecadca9fe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fc8884f79 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e2f52ee4-55f1-410a-87ae-aa39809ae009 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fcd035802 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: 9c8f1704-171b-4427-8531-440db2aee682 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fcd778853 namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: 132501bd-fcda-4854-8bed-d6e27d2e8d20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:11:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:35Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7a9fe996452b namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: fb6c67a5-1391-4e63-9c75-d7f760330709 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7b2314b44b2d namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: 67b30706-5f42-4e47-a5a8-936f8c54d9be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:11:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:11:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:33Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7b2314b4d143 namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: fd220e28-32ed-4c19-92f9-5d220ccf520d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:11:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 2a63b743-a196-48a1-a6cb-dd7dcb38efad kind: Event lastTimestamp: "2026-07-02T13:11:35Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:11:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:11:35Z" name: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw.18be7b23b221d313 namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: fd594c8a-a903-4d55-8bea-6ff368f90ba5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b2e72-predictor-6cbff7944f namespace: kserve-ci-e2e-test resourceVersion: "21472" uid: 40cbdc17-5436-4061-b9c3-53e178747bef kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: 'Created pod: success-200-isvc-b2e72-predictor-6cbff7944f-jcljw' metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:08Z" name: success-200-isvc-b2e72-predictor-6cbff7944f.18be7a9f845b8ed2 namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 0414fd7b-2991-4c3d-9e62-816f0e24f383 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21468" uid: 07adbd79-9f78-48d7-bd9b-b13f9d9344cf kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: Scaled up replica set success-200-isvc-b2e72-predictor-6cbff7944f from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:07Z" name: success-200-isvc-b2e72-predictor.18be7a9f7e222b7f namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: deac7534-8222-4fac-9e37-47be39b388b3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 96399f2f-629d-4510-a7f6-f4e58ac5d2d2 kind: Event lastTimestamp: "2026-07-02T13:02:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:37Z" name: success-200-isvc-b2e72-predictor.18be7aa2fd5f1598 namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: b8c3c288-dd1b-4b7d-abab-09d25f70486b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:02:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: 96399f2f-629d-4510-a7f6-f4e58ac5d2d2 kind: Event lastTimestamp: "2026-07-02T13:02:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:02:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:02:37Z" name: success-200-isvc-b2e72-predictor.18be7aa2fdccdefa namespace: kserve-ci-e2e-test resourceVersion: "21825" uid: e0ffa8b1-b961-458f-8abc-046756e4d347 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 96399f2f-629d-4510-a7f6-f4e58ac5d2d2 kind: Event lastTimestamp: "2026-07-02T13:03:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:38Z" name: success-200-isvc-b2e72-predictor.18be7aa9fcd705d7 namespace: kserve-ci-e2e-test resourceVersion: "22550" uid: f87dc06e-2046-4e47-9638-4c2d27e17a6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:02:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b2e72-predictor namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 96399f2f-629d-4510-a7f6-f4e58ac5d2d2 kind: Event lastTimestamp: "2026-07-02T13:03:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:02:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:03:38Z" name: success-200-isvc-b2e72-predictor.18be7aa9fdb6dc4e namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: 20cc6ee7-2f0c-4727-b499-8e644b419b6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: e36d3d97-cb31-4346-86a5-20fbd999dc5b kind: Event lastTimestamp: "2026-07-02T13:03:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:05Z" name: success-200-isvc-b2e72.18be7a9f7e8fde42 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: e324d7cd-f861-4c64-bc05-23b0fbe8c0cd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: e36d3d97-cb31-4346-86a5-20fbd999dc5b kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-b2e72": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2e72": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:07Z" name: success-200-isvc-b2e72.18be7a9f7f20860b namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: e3f9a3d4-7fb5-478f-8a45-37a2ddc6cd2b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "21448" uid: e36d3d97-cb31-4346-86a5-20fbd999dc5b kind: Event lastTimestamp: "2026-07-02T13:02:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b2e72": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:02:07Z" name: success-200-isvc-b2e72.18be7a9f7f20dbe2 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 01e4c44a-5322-4c40-b7c0-d89613fa7856 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:03:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b2e72 namespace: kserve-ci-e2e-test resourceVersion: "22110" uid: e36d3d97-cb31-4346-86a5-20fbd999dc5b kind: Event lastTimestamp: "2026-07-02T13:03:05Z" message: InferenceService [success-200-isvc-b2e72] is Ready metadata: creationTimestamp: "2026-07-02T13:03:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:03:05Z" name: success-200-isvc-b2e72.18be7aacf8a5245b namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 3e91ec4c-5b24-482a-9db2-bb2e0eb1d0a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:00:57.769681Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c0346-predictor-848ccd5d77-kzr6z to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f28dba0b9 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: b7abef66-b1b0-4667-94b1-93ffc6dee4c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20604" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f42d7381a namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: 31ff5b7b-9c4c-4c07-af32-c52cd9eb4f15 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f44533163 namespace: kserve-ci-e2e-test resourceVersion: "20656" uid: 5b8df912-b293-4609-973a-aca803f5bb30 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f4741c7de namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 60039c8d-43be-4e03-af6d-a2a3b82d06c0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f47ba621b namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: 6e0ac148-a265-4166-aa9c-a0c8ff6ec435 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f47c7a6e5 namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: d23245c2-6ba0-49e3-8a90-1b794b47ff2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f4ab21bf2 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: b16c982f-09e5-4714-ba31-50cba40c63f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:00:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:58Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f4b3d0bf8 namespace: kserve-ci-e2e-test resourceVersion: "20667" uid: 4989642c-8b2c-4f8c-b04b-df866e0b0f30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:01:34Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:34Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a8f4e3fe3af namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 7e95b81a-e35d-44d1-a6a2-23149191bef3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:08Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a9f8e8186fd namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 63e2638d-4aaf-4a75-b631-9b5004d03af9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:02:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:02:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:08Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a9f8e8202a0 namespace: kserve-ci-e2e-test resourceVersion: "21523" uid: 95056566-045e-4a64-8985-acad318536e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:02:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 1b200e0e-cf4f-450b-8b90-57be599a90d3 kind: Event lastTimestamp: "2026-07-02T13:02:09Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:02:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:02:09Z" name: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z.18be7a9fd6abd27b namespace: kserve-ci-e2e-test resourceVersion: "21573" uid: f4b5ddc4-b5d9-4b86-ab78-7ab9f8191b8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c0346-predictor-848ccd5d77 namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 90b66c9b-505a-4d41-9567-95bcbeffcb27 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: 'Created pod: success-200-isvc-c0346-predictor-848ccd5d77-kzr6z' metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-c0346-predictor-848ccd5d77.18be7a8f285b0977 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: cc03e4b4-0377-4a31-a7e6-a9eba4f62d11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20569" uid: f511d37c-c29e-4a56-89bb-0044521190eb kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Scaled up replica set success-200-isvc-c0346-predictor-848ccd5d77 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-c0346-predictor.18be7a8f233031f1 namespace: kserve-ci-e2e-test resourceVersion: "20573" uid: 05a8b65b-6f46-4292-a3b2-feeb8812f1d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 916cfddd-7af5-4e34-b758-289179ef7d34 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: success-200-isvc-c0346-predictor.18be7a92a1e5f3af namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: c0f59fd3-b56b-438f-808f-a21f41b834c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-02T13:01:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20570" uid: 916cfddd-7af5-4e34-b758-289179ef7d34 kind: Event lastTimestamp: "2026-07-02T13:01:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:01:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:57Z" name: success-200-isvc-c0346-predictor.18be7a92a25bbed4 namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: 174d1e61-aecb-4e2f-a4e6-f5dcc0208e1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 916cfddd-7af5-4e34-b758-289179ef7d34 kind: Event lastTimestamp: "2026-07-02T13:01:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:27Z" name: success-200-isvc-c0346-predictor.18be7a9621310027 namespace: kserve-ci-e2e-test resourceVersion: "20913" uid: 3ed5efd1-c401-4a62-b2cc-3f1017c870c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0346-predictor namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 916cfddd-7af5-4e34-b758-289179ef7d34 kind: Event lastTimestamp: "2026-07-02T13:01:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:01:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:01:27Z" name: success-200-isvc-c0346-predictor.18be7a9621c0807a namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: 0e63cbcb-4434-4bb3-8045-2ae91b2b96c7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: a9d36cf0-d34f-47e6-ab4a-df8a24ac6868 kind: Event lastTimestamp: "2026-07-02T13:01:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:44Z" name: success-200-isvc-c0346.18be7a8f233000df namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 597fb988-3e72-4d38-ba64-e31ad0133ae7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: a9d36cf0-d34f-47e6-ab4a-df8a24ac6868 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: 'Failed to update status for InferenceService "success-200-isvc-c0346": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c0346": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-c0346.18be7a8f2419d1a6 namespace: kserve-ci-e2e-test resourceVersion: "20577" uid: 4fd63475-df62-4f87-b89d-ba37da949e44 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "20561" uid: a9d36cf0-d34f-47e6-ab4a-df8a24ac6868 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c0346": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:00:57Z" name: success-200-isvc-c0346.18be7a8f241a18a3 namespace: kserve-ci-e2e-test resourceVersion: "20580" uid: 31109132-a0d9-40ae-8a8c-5bdb3beeb922 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:01:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0346 namespace: kserve-ci-e2e-test resourceVersion: "21049" uid: a9d36cf0-d34f-47e6-ab4a-df8a24ac6868 kind: Event lastTimestamp: "2026-07-02T13:01:44Z" message: InferenceService [success-200-isvc-c0346] is Ready metadata: creationTimestamp: "2026-07-02T13:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:01:44Z" name: success-200-isvc-c0346.18be7a9a09cd6aa9 namespace: kserve-ci-e2e-test resourceVersion: "21050" uid: aa0db200-a339-4d7f-84bc-19ed60df4a62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:12:22.937362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26738" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7674-predictor-84c5db9945-fr67z to ip-10-0-137-127.ec2.internal metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:12:22Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2eb009f070 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 37e734d2-5e34-4254-9adf-6e53f2459f5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2eca5c6137 namespace: kserve-ci-e2e-test resourceVersion: "26806" uid: e2faf967-4e45-4737-b72a-1e23326dfd1c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1694" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ecbd3f5de namespace: kserve-ci-e2e-test resourceVersion: "26809" uid: e830ccaa-9a48-4038-9fcb-cb38b1b6102c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ecf2ae063 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 3269a1ac-7895-47b2-997f-7a24e837ac05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ecfa25f81 namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: 2f63a4fb-0d2b-480d-9fee-f20f1708487b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ecfadf87c namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: d67de8b0-c5d4-4c60-be7a-386783be0617 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ed3dd7025 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: a77b90d6-3f58-4681-ab47-99a4bf760b9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:12:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:12:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:12:23Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ed4552f67 namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 0f3449c9-4bbe-403b-b512-29587640f3cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-02T13:12:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:13:10Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:12:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:10Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7b2ef8d6337c namespace: kserve-ci-e2e-test resourceVersion: "27407" uid: 20c6880c-3bef-4c2c-906c-ed1e462efe5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:29:52Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-02T13:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:52Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7c2304f91f97 namespace: kserve-ci-e2e-test resourceVersion: "34521" uid: 24745dac-3484-401f-867a-73372f5896c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:29:52Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-02T13:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:52Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7c2304f9a532 namespace: kserve-ci-e2e-test resourceVersion: "34522" uid: 80a645a1-e695-4378-a248-598357a1c819 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7674-predictor-84c5db9945-fr67z namespace: kserve-ci-e2e-test resourceVersion: "26743" uid: 11d14f5a-762b-4186-abe5-0d694152b0d3 kind: Event lastTimestamp: "2026-07-02T13:29:55Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-02T13:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:55Z" name: success-200-isvc-f7674-predictor-84c5db9945-fr67z.18be7c23add28826 namespace: kserve-ci-e2e-test resourceVersion: "34592" uid: 6fa8e85b-c0bc-4626-8bb5-6b7088775763 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-127.ec2.internal source: component: kubelet host: ip-10-0-137-127.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7674-predictor-84c5db9945 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: dc46e3f6-1145-4b81-a51f-2b6efd75df6e kind: Event lastTimestamp: "2026-07-02T13:12:22Z" message: 'Created pod: success-200-isvc-f7674-predictor-84c5db9945-fr67z' metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:22Z" name: success-200-isvc-f7674-predictor-84c5db9945.18be7b2eaf86ce1c namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: ebc15964-6360-4b63-bb63-0de615934866 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26717" uid: 5ec4745b-e6e1-4a5a-858b-3c8d3fe2c017 kind: Event lastTimestamp: "2026-07-02T13:12:22Z" message: Scaled up replica set success-200-isvc-f7674-predictor-84c5db9945 from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:22Z" name: success-200-isvc-f7674-predictor.18be7b2eabed2c4c namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 0deaa8cb-2601-46b7-b36d-ca0ff6e02a23 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 373f26b4-5204-410a-ad7d-056c10d6aafc kind: Event lastTimestamp: "2026-07-02T13:12:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:52Z" name: success-200-isvc-f7674-predictor.18be7b322abba54f namespace: kserve-ci-e2e-test resourceVersion: "27260" uid: 8d7e8b9c-7c4c-4c65-bc58-e464cb8bc303 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:12:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 373f26b4-5204-410a-ad7d-056c10d6aafc kind: Event lastTimestamp: "2026-07-02T13:12:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:12:52Z" name: success-200-isvc-f7674-predictor.18be7b322b801038 namespace: kserve-ci-e2e-test resourceVersion: "27261" uid: c285a639-16f4-4e74-b21d-8ccf0adbe266 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:13:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 373f26b4-5204-410a-ad7d-056c10d6aafc kind: Event lastTimestamp: "2026-07-02T13:13:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:53Z" name: success-200-isvc-f7674-predictor.18be7b392a4b3e36 namespace: kserve-ci-e2e-test resourceVersion: "27851" uid: 0a22bb82-3bcb-4ece-a1cf-057a1c882e95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-02T13:13:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7674-predictor namespace: kserve-ci-e2e-test resourceVersion: "26992" uid: 373f26b4-5204-410a-ad7d-056c10d6aafc kind: Event lastTimestamp: "2026-07-02T13:13:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-02T13:13:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:53Z" name: success-200-isvc-f7674-predictor.18be7b392ae912ab namespace: kserve-ci-e2e-test resourceVersion: "27852" uid: c8ace612-02a2-403d-9b24-a2cede523df6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:12:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "26710" uid: 09a8436a-5f9e-4632-aa65-9c8ab1c22941 kind: Event lastTimestamp: "2026-07-02T13:13:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-02T13:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:20Z" name: success-200-isvc-f7674.18be7b2eac371677 namespace: kserve-ci-e2e-test resourceVersion: "27499" uid: f52c6fdb-c6fd-4587-84a8-7b84ff9fd6ad reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27497" uid: 09a8436a-5f9e-4632-aa65-9c8ab1c22941 kind: Event lastTimestamp: "2026-07-02T13:13:20Z" message: InferenceService [success-200-isvc-f7674] is Ready metadata: creationTimestamp: "2026-07-02T13:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:20Z" name: success-200-isvc-f7674.18be7b3c0bdb5a05 namespace: kserve-ci-e2e-test resourceVersion: "27498" uid: 0bbbe03c-a107-4e32-bb7e-5c4762c4e25d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:52:42.873352Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-04e3b-866d4fb65f-xqsz9 to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T12:52:42Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:52:42Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a1beebcf021 namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: 3c3a670c-6b69-4488-b850-2adfde3fe6b6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:43Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T12:52:43Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:52:43Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a1c07abc67a namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: c0147b76-468f-42b9-b9af-acf9cfeab963 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04e3b} kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T12:52:43Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T12:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:43Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a1c08fc304b namespace: kserve-ci-e2e-test resourceVersion: "17049" uid: 948334ce-b492-4c64-9573-55decf0d44c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04e3b} kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T12:52:43Z" message: 'Created container: switch-graph-04e3b' metadata: creationTimestamp: "2026-07-02T12:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:43Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a1c0b95ce38 namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: a4423eb1-3d9e-454c-bd37-d96d9dc4c601 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04e3b} kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T12:52:43Z" message: Started container switch-graph-04e3b metadata: creationTimestamp: "2026-07-02T12:52:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:43Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a1c0bf9453d namespace: kserve-ci-e2e-test resourceVersion: "17051" uid: 1267e7c5-3de9-4c0e-b898-f01bf560f976 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04e3b} kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T13:00:57Z" message: Stopping container switch-graph-04e3b metadata: creationTimestamp: "2026-07-02T13:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:00:57Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a8f15a0a94e namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: e0ec7ede-47e5-4cac-a0a5-9ded03807537 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T13:00:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-04e3b} kind: Pod name: switch-graph-04e3b-866d4fb65f-xqsz9 namespace: kserve-ci-e2e-test resourceVersion: "17028" uid: 8999b35d-3e41-4baa-928c-cdbeef1a73f5 kind: Event lastTimestamp: "2026-07-02T13:01:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:00:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:01:24Z" name: switch-graph-04e3b-866d4fb65f-xqsz9.18be7a8fa8f0b593 namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: b9a26228-73a1-48d5-9960-b89c471f0c61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:42Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-04e3b-866d4fb65f namespace: kserve-ci-e2e-test resourceVersion: "17021" uid: 7fca53bc-ee10-4bd1-97c0-bfd74f412a6d kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: 'Created pod: switch-graph-04e3b-866d4fb65f-xqsz9' metadata: creationTimestamp: "2026-07-02T12:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: switch-graph-04e3b-866d4fb65f.18be7a1bee40f868 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 1912ccc1-8a07-4e3a-90e7-9c6b7f10e740 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:42Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-04e3b namespace: kserve-ci-e2e-test resourceVersion: "17019" uid: c91e2e92-c401-493f-a72b-cb252e5365e4 kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: Scaled up replica set switch-graph-04e3b-866d4fb65f from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:52:42Z" name: switch-graph-04e3b.18be7a1bed51c9bf namespace: kserve-ci-e2e-test resourceVersion: "17022" uid: 3fc2beaa-0305-435c-9688-4fda11945c42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:52:42Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-04e3b namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: dce8f648-f973-46bf-9784-1df841b7a7a3 kind: Event lastTimestamp: "2026-07-02T12:52:42Z" message: InferenceGraph [switch-graph-04e3b] is Ready metadata: creationTimestamp: "2026-07-02T12:52:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:52:42Z" name: switch-graph-04e3b.18be7a1bf19a7efa namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: ed8e3884-2392-486e-84ca-7e1d09220fcf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04e3b namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 74260367-ceaf-4af9-8002-5b74d31078dd kind: Event lastTimestamp: "2026-07-02T12:53:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:12Z" name: switch-graph-04e3b.18be7a1f6cdb6a31 namespace: kserve-ci-e2e-test resourceVersion: "17298" uid: f50f884e-7c06-44e1-8bb7-d567b12ec0ff reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T12:52:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-04e3b namespace: kserve-ci-e2e-test resourceVersion: "17020" uid: 74260367-ceaf-4af9-8002-5b74d31078dd kind: Event lastTimestamp: "2026-07-02T12:53:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T12:52:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:53:12Z" name: switch-graph-04e3b.18be7a1f6dabd94d namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: 7371cb0c-9975-46de-b138-38e87a32ffde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-02T12:51:32.498981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-75b89-6449db88c5-qt9lc to ip-10-0-140-7.ec2.internal metadata: creationTimestamp: "2026-07-02T12:51:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T12:51:32Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0b8c193f58 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: b358133e-de08-488a-9076-b6291dda0957 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-75b89-serving-cert" not found' metadata: creationTimestamp: "2026-07-02T12:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:32Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0b96ba08a7 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: b1eff612-ac13-4f4e-a598-deb407f126d1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:33Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T12:51:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T12:51:33Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0bc99decf9 namespace: kserve-ci-e2e-test resourceVersion: "16037" uid: e3c0776a-9600-401c-96ce-3e1f9d420c53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:33Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" metadata: creationTimestamp: "2026-07-02T12:51:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:33Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0bcaf7028c namespace: kserve-ci-e2e-test resourceVersion: "16039" uid: 5c841ded-252f-417b-8802-71e99e31225d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" in 2.237s (2.237s including waiting). Image size: 218070116 bytes.' metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0c50556b74 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: a4349791-65e2-4237-b70c-ac54c2d1b022 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: 'Created container: switch-graph-75b89' metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0c534c918a namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 7941a8d1-2034-4c42-9e64-d4923b833b69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:35Z" message: Started container switch-graph-75b89 metadata: creationTimestamp: "2026-07-02T12:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:35Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0c53b2e941 namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 1c5e352b-807b-4493-9786-af838a6e1996 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:51:42Z" message: Stopping container switch-graph-75b89 metadata: creationTimestamp: "2026-07-02T12:51:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:51:42Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0de7d5d1ef namespace: kserve-ci-e2e-test resourceVersion: "16141" uid: 14cf3cbf-832d-438a-b348-d3086efa6d75 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-02T12:51:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-75b89} kind: Pod name: switch-graph-75b89-6449db88c5-qt9lc namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: a689b081-9855-4c37-a268-7bfa6c95ce35 kind: Event lastTimestamp: "2026-07-02T12:52:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T12:51:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T12:52:12Z" name: switch-graph-75b89-6449db88c5-qt9lc.18be7a0ef803db10 namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: cb877b03-9a6b-42fe-968b-da8136faf1a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-7.ec2.internal source: component: kubelet host: ip-10-0-140-7.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-75b89-6449db88c5 namespace: kserve-ci-e2e-test resourceVersion: "16009" uid: c1b54f8c-1d2e-4474-9a90-d8a169a09096 kind: Event lastTimestamp: "2026-07-02T12:51:32Z" message: 'Created pod: switch-graph-75b89-6449db88c5-qt9lc' metadata: creationTimestamp: "2026-07-02T12:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:32Z" name: switch-graph-75b89-6449db88c5.18be7a0b8bb7a002 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 79710aca-2da6-4848-8149-38e145359452 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-75b89 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 72316dcc-5220-421e-8ba2-07df469f9151 kind: Event lastTimestamp: "2026-07-02T12:51:32Z" message: Scaled up replica set switch-graph-75b89-6449db88c5 from 0 to 1 metadata: creationTimestamp: "2026-07-02T12:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T12:51:32Z" name: switch-graph-75b89.18be7a0b8af12829 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: e10b788c-b4e0-420a-8386-3c0c8dc7efce reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T12:51:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-75b89 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: c8a3cfea-6b0c-45bd-8bfd-e86b94570080 kind: Event lastTimestamp: "2026-07-02T12:51:32Z" message: InferenceGraph [switch-graph-75b89] is Ready metadata: creationTimestamp: "2026-07-02T12:51:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T12:51:32Z" name: switch-graph-75b89.18be7a0b8f9c449a namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 61886b74-352b-42ab-bbc5-c347db07bfdd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-02T13:13:33.113025Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27627" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f7674-7d596db84c-9969j to ip-10-0-128-177.ec2.internal metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c-9969j.18be7b3f06d456c9 namespace: kserve-ci-e2e-test resourceVersion: "27633" uid: 01b25b16-3328-4bae-a449-49abd7a416b3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-56747c54b6-mthzj source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27636" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c-9969j.18be7b3f1fedfb66 namespace: kserve-ci-e2e-test resourceVersion: "27647" uid: 28365dfe-d9e4-4491-bed3-e2e267befea6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f7674} kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:03660399b01ea350e32a900800c331d43d890f82d1c053ba75b58e21480c1066" already present on machine metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c-9969j.18be7b3f228457ff namespace: kserve-ci-e2e-test resourceVersion: "27650" uid: b4842e19-238a-4baf-ba66-d7413dbcadbd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f7674} kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: 'Created container: switch-graph-f7674' metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c-9969j.18be7b3f256be332 namespace: kserve-ci-e2e-test resourceVersion: "27651" uid: f93a99e2-8e54-4aa4-8076-329667b05aac reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f7674} kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: Started container switch-graph-f7674 metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c-9969j.18be7b3f25d56ec7 namespace: kserve-ci-e2e-test resourceVersion: "27652" uid: af4b7843-db09-4e67-ab1f-87cece194dec reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:29:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f7674} kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:29:52Z" message: Stopping container switch-graph-f7674 metadata: creationTimestamp: "2026-07-02T13:29:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:29:52Z" name: switch-graph-f7674-7d596db84c-9969j.18be7c22fbed88a0 namespace: kserve-ci-e2e-test resourceVersion: "34505" uid: c1c46f49-bd63-4de8-8f63-132dc5eb78d0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-02T13:29:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f7674} kind: Pod name: switch-graph-f7674-7d596db84c-9969j namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 5ef42e84-c1c9-4b05-bc8d-9403af30bf3f kind: Event lastTimestamp: "2026-07-02T13:30:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-02T13:29:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-02T13:30:15Z" name: switch-graph-f7674-7d596db84c-9969j.18be7c23bc3d3cfd namespace: kserve-ci-e2e-test resourceVersion: "34744" uid: da481fde-e5f5-4707-81db-302db19c956b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-177.ec2.internal source: component: kubelet host: ip-10-0-128-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f7674-7d596db84c namespace: kserve-ci-e2e-test resourceVersion: "27621" uid: 53839056-717f-4734-be43-118a118a0c76 kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: 'Created pod: switch-graph-f7674-7d596db84c-9969j' metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674-7d596db84c.18be7b3f0583dc1c namespace: kserve-ci-e2e-test resourceVersion: "27628" uid: ecb22329-7e82-43d1-a532-c925c2b49ce1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27618" uid: 65f4cd0e-ee0b-48ac-90f1-a32de6305911 kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: Scaled up replica set switch-graph-f7674-7d596db84c from 0 to 1 metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674.18be7b3f0365a778 namespace: kserve-ci-e2e-test resourceVersion: "27626" uid: 58a521db-29cf-4bbd-8331-9f4f5e81bed4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-02T13:13:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: 40206623-b25b-4a6c-8c75-e14b44943592 kind: Event lastTimestamp: "2026-07-02T13:13:33Z" message: InferenceGraph [switch-graph-f7674] is Ready metadata: creationTimestamp: "2026-07-02T13:13:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-02T13:13:33Z" name: switch-graph-f7674.18be7b3f12732f87 namespace: kserve-ci-e2e-test resourceVersion: "27641" uid: a520b176-23a3-4d6d-b88a-e2a4f90f9ff9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27620" uid: f93611a0-b137-4e2c-8a49-a8a2a0a6b53c kind: Event lastTimestamp: "2026-07-02T13:14:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:14:03Z" name: switch-graph-f7674.18be7b4281fd3c28 namespace: kserve-ci-e2e-test resourceVersion: "27925" uid: d2e4be4b-5404-461a-9e5d-4a91cf57a69c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-02T13:13:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-f7674 namespace: kserve-ci-e2e-test resourceVersion: "27620" uid: f93611a0-b137-4e2c-8a49-a8a2a0a6b53c kind: Event lastTimestamp: "2026-07-02T13:14:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-02T13:13:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-02T13:14:03Z" name: switch-graph-f7674.18be7b42827fe596 namespace: kserve-ci-e2e-test resourceVersion: "27926" uid: c3263904-7fe7-44c4-a187-c4073a5250fb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34779"