--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:05.637547Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-02c81-7c5f497f56-bxss9 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:05Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:05Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650ee0c4953da namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 11778a59-c475-476a-b43f-5bf6ab3b4aad reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:06Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:06Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650ee2658da45 namespace: kserve-ci-e2e-test resourceVersion: "21253" uid: 0eff9ed0-0618-4ad2-b6a2-7b09bdb1feef reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02c81} kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:06Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650ee2a3c02be namespace: kserve-ci-e2e-test resourceVersion: "21255" uid: c79b5b9a-f26a-4bef-ad69-272173cea767 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02c81} kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:06Z" message: 'Created container: ensemble-graph-02c81' metadata: creationTimestamp: "2026-07-28T01:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:06Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650ee2d9c1bb3 namespace: kserve-ci-e2e-test resourceVersion: "21256" uid: 6762ff57-d75e-477e-8068-0eb5445079a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02c81} kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:06Z" message: Started container ensemble-graph-02c81 metadata: creationTimestamp: "2026-07-28T01:48:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:06Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650ee2e1a5db0 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 960cd38b-8264-4f74-9e44-cc913d59da53 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02c81} kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Stopping container ensemble-graph-02c81 metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:15Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650f0648825bc namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 412fbb59-71a8-43ed-9475-bd0e81799fe5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-02c81} kind: Pod name: ensemble-graph-02c81-7c5f497f56-bxss9 namespace: kserve-ci-e2e-test resourceVersion: "21233" uid: d0503ca4-cb66-4d74-81a9-77e30998df69 kind: Event lastTimestamp: "2026-07-28T01:48:42Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:42Z" name: ensemble-graph-02c81-7c5f497f56-bxss9.18c650f0c1e1d5a3 namespace: kserve-ci-e2e-test resourceVersion: "21752" uid: 5b3cb44f-506b-407f-a1b9-a5d446ba4c31 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:05Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-02c81-7c5f497f56 namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: 3c52dde7-e6da-493b-b720-bee39c8fbdd8 kind: Event lastTimestamp: "2026-07-28T01:48:05Z" message: 'Created pod: ensemble-graph-02c81-7c5f497f56-bxss9' metadata: creationTimestamp: "2026-07-28T01:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:05Z" name: ensemble-graph-02c81-7c5f497f56.18c650ee0bd885d3 namespace: kserve-ci-e2e-test resourceVersion: "21232" uid: c47a030e-187d-4c08-bb4f-5380855ace27 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:05Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-02c81 namespace: kserve-ci-e2e-test resourceVersion: "21226" uid: 92ee35a5-207c-438f-ae4a-28ce87a9aae6 kind: Event lastTimestamp: "2026-07-28T01:48:05Z" message: Scaled up replica set ensemble-graph-02c81-7c5f497f56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:05Z" name: ensemble-graph-02c81.18c650ee0ab7e282 namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 483c934b-978b-40b6-91aa-cc692cedd2a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:05Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-02c81 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 9802841a-dad2-4c9c-a8d2-0fa9c25705d4 kind: Event lastTimestamp: "2026-07-28T01:48:05Z" message: InferenceGraph [ensemble-graph-02c81] is Ready metadata: creationTimestamp: "2026-07-28T01:48:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:05Z" name: ensemble-graph-02c81.18c650ee115f69fd namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: ff678899-4575-4b68-a10c-adf609b1e762 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:49:25.955072Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22277" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-5a4a0-995768857-2rw89 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:49:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:49:25Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65100bf9587a1 namespace: kserve-ci-e2e-test resourceVersion: "22282" uid: 60d11e49-93a6-451e-bd52-f8ee50d11a99 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:49:26Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-5a4a0-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:26Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65100ccb94d7b namespace: kserve-ci-e2e-test resourceVersion: "22297" uid: 56b19742-c5db-43cb-b0de-d783793c730b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:49:26Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:49:26Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65100fd75a43e namespace: kserve-ci-e2e-test resourceVersion: "22306" uid: d944d155-29b0-46dc-949d-f5fc96def1ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5a4a0} kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:49:27Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:27Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65100fedacf36 namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: 83ebd41f-278c-4c66-abff-5ed729d0c9ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5a4a0} kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:49:27Z" message: 'Created container: ensemble-graph-5a4a0' metadata: creationTimestamp: "2026-07-28T01:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:27Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c6510102f959a5 namespace: kserve-ci-e2e-test resourceVersion: "22309" uid: db6104f5-0be8-4445-9eb7-5fe13e039202 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5a4a0} kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:49:27Z" message: Started container ensemble-graph-5a4a0 metadata: creationTimestamp: "2026-07-28T01:49:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:27Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c6510103baf54b namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: 2dc59d46-6412-45be-ae5d-61408942d86e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5a4a0} kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Stopping container ensemble-graph-5a4a0 metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:40Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65173e53225d3 namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 14adda34-d947-43b9-aa4e-5ba2ae7395b1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:57:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-5a4a0} kind: Pod name: ensemble-graph-5a4a0-995768857-2rw89 namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: d12f1df4-84a9-4ad8-a8c2-c0fbbaca4ece kind: Event lastTimestamp: "2026-07-28T01:58:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:08Z" name: ensemble-graph-5a4a0-995768857-2rw89.18c65174a083bf68 namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: c1420ec4-d131-43a4-8d26-7af8350c728a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-5a4a0-995768857 namespace: kserve-ci-e2e-test resourceVersion: "22274" uid: 1f1e11b0-ba4d-42f0-8044-672160368938 kind: Event lastTimestamp: "2026-07-28T01:49:25Z" message: 'Created pod: ensemble-graph-5a4a0-995768857-2rw89' metadata: creationTimestamp: "2026-07-28T01:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:25Z" name: ensemble-graph-5a4a0-995768857.18c65100bf40043c namespace: kserve-ci-e2e-test resourceVersion: "22279" uid: c819eaef-fb9a-458b-be62-ef4416496c27 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22272" uid: 9f918ee6-4e28-4a6f-9b09-6b3776432b27 kind: Event lastTimestamp: "2026-07-28T01:49:25Z" message: Scaled up replica set ensemble-graph-5a4a0-995768857 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:49:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:25Z" name: ensemble-graph-5a4a0.18c65100be7908bc namespace: kserve-ci-e2e-test resourceVersion: "22275" uid: fdb7e974-0d66-4a64-98d1-b8b30ef40945 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22288" uid: eabfb91f-89b1-4206-910c-c1e6e9b0fa0b kind: Event lastTimestamp: "2026-07-28T01:49:26Z" message: InferenceGraph [ensemble-graph-5a4a0] is Ready metadata: creationTimestamp: "2026-07-28T01:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:26Z" name: ensemble-graph-5a4a0.18c65100c35c4e68 namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 489c8c6e-2c0e-4604-992b-51085dad1af0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: eabfb91f-89b1-4206-910c-c1e6e9b0fa0b kind: Event lastTimestamp: "2026-07-28T01:49:26Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-5a4a0": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-5a4a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:26Z" name: ensemble-graph-5a4a0.18c65100c5b638b5 namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: 4d77c168-acf1-41d9-a222-33360eab46b1 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: eabfb91f-89b1-4206-910c-c1e6e9b0fa0b kind: Event lastTimestamp: "2026-07-28T01:49:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-5a4a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:49:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:26Z" name: ensemble-graph-5a4a0.18c65100c5b69c2c namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 3470d3f4-029c-4b7c-984a-7511638f41b2 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: 390f65a9-2166-489a-9464-930d1b316521 kind: Event lastTimestamp: "2026-07-28T01:49:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:40Z" name: ensemble-graph-5a4a0.18c651043d3a3237 namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: 8533b481-1533-4d51-8e68-be775dcc355b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22273" uid: 390f65a9-2166-489a-9464-930d1b316521 kind: Event lastTimestamp: "2026-07-28T01:49:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:40Z" name: ensemble-graph-5a4a0.18c651043d9cbda3 namespace: kserve-ci-e2e-test resourceVersion: "22437" uid: 0e172b09-85d8-45c6-96ee-05883c5a692a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 390f65a9-2166-489a-9464-930d1b316521 kind: Event lastTimestamp: "2026-07-28T01:49:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:55Z" name: ensemble-graph-5a4a0.18c65107bc407fd6 namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: 5fc02bb1-7ee3-4941-b7ff-53a6a7eefbbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 390f65a9-2166-489a-9464-930d1b316521 kind: Event lastTimestamp: "2026-07-28T01:49:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:55Z" name: ensemble-graph-5a4a0.18c65107bc9ce5a9 namespace: kserve-ci-e2e-test resourceVersion: "22587" uid: 6f981f43-b3f2-4242-8e0e-a5e1bc7f5372 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:46:55.669421Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20441" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650ddc1dc88d7 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: 35d8693e-8085-4c97-a2ba-78797d39001c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dddb657ca8 namespace: kserve-ci-e2e-test resourceVersion: "20482" uid: 6b00c34a-7a98-4547-811e-be0991180c93 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dddcc0094b namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 27a1a49a-9bc1-4c6a-a9df-e0efb47252bf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dddf869d7c namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: 9a982b12-19d7-49b4-b992-4f4544b52988 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dddff9770b namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 45ab46af-cc5a-4855-b435-cd818645f936 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dde002d361 namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: ad9c467b-80bc-47bc-a492-5f5a758e868c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dde40c1c41 namespace: kserve-ci-e2e-test resourceVersion: "20493" uid: 5422be4a-6f4f-4d4d-a418-e3cc5086dda9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650dde4d919dc namespace: kserve-ci-e2e-test resourceVersion: "20494" uid: 56882553-0905-42cc-a01c-41fd9a3bfbad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:46:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:47:33Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:46:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:33Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650de3ca21ba5 namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: 73ab8188-8ffd-4fe8-a5b1-329f9c7ae360 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650f06e9c6a68 namespace: kserve-ci-e2e-test resourceVersion: "21412" uid: c710eb86-2db3-492d-bedb-eb984ace670d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650f06e9ce62a namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: a0bded34-473c-4eed-bb62-e0344b05b0f9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 8b436056-36ce-43e0-b251-495797c91744 kind: Event lastTimestamp: "2026-07-28T01:48:18Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:18Z" name: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6.18c650f118cf7b56 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 26b2bc81-3c0f-433c-9d63-2a87910aa579 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-02c81-predictor-58cbbbb866 namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 29d7fe9b-8ad5-4dab-a140-ae509fe8abb0 kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'Created pod: error-404-isvc-02c81-predictor-58cbbbb866-f6tf6' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-02c81-predictor-58cbbbb866.18c650ddc16ba308 namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 90d9524c-e0fb-4f2b-99ad-4ad24e58992d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: e9913f7d-3e21-4513-b4bc-3ad9724c94ae kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Scaled up replica set error-404-isvc-02c81-predictor-58cbbbb866 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-02c81-predictor.18c650ddbf9648fc namespace: kserve-ci-e2e-test resourceVersion: "20433" uid: cb160f45-18f9-4aba-a34b-ac597fd765e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 23809c21-1534-4b75-86bd-a8dc3545bd0f kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-02c81-predictor.18c650e13de8d2d3 namespace: kserve-ci-e2e-test resourceVersion: "20846" uid: fef13556-2131-413d-8498-3791cd7fc732 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 23809c21-1534-4b75-86bd-a8dc3545bd0f kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-02c81-predictor.18c650e13e28bf1f namespace: kserve-ci-e2e-test resourceVersion: "20847" uid: 8b183d4f-633c-4f47-92cd-83acf25f0657 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 23809c21-1534-4b75-86bd-a8dc3545bd0f kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: error-404-isvc-02c81-predictor.18c650e83b6dca35 namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: c96db13c-9c3a-4065-8264-c2971234ad55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20627" uid: 23809c21-1534-4b75-86bd-a8dc3545bd0f kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: error-404-isvc-02c81-predictor.18c650e83bd05a3c namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: e3e05e24-b412-40c9-96eb-143d66a80079 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: de338822-be69-4832-a61b-0e4ee30dd21f kind: Event lastTimestamp: "2026-07-28T01:47:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:43Z" name: error-404-isvc-02c81.18c650ddbf9ccbf6 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 2ec8925a-aeb4-413f-ba00-3c60ff9fc80b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: de338822-be69-4832-a61b-0e4ee30dd21f kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'Failed to update status for InferenceService "error-404-isvc-02c81": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02c81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-02c81.18c650ddc02f556b namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: d898dfad-e105-4ad3-9fd3-3b0060d2b676 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: de338822-be69-4832-a61b-0e4ee30dd21f kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-02c81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-02c81.18c650ddc02fa683 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: dcd6fd01-d5fa-41ae-8dd4-a03244196214 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: de338822-be69-4832-a61b-0e4ee30dd21f kind: Event lastTimestamp: "2026-07-28T01:47:43Z" message: InferenceService [error-404-isvc-02c81] is Ready metadata: creationTimestamp: "2026-07-28T01:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:43Z" name: error-404-isvc-02c81.18c650e8f7a9533b namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: d2798755-9931-45f1-949a-3908635b35f1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:47:25.445115Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-2a946-predictor-85b9467f54-9t9hc to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4b0a192c2 namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 01b725e1-3eb6-4b72-811a-3c9dcafd44ba reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-2a946-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4bb0b32fa namespace: kserve-ci-e2e-test resourceVersion: "20844" uid: a13674b0-6913-4067-818d-bd41ea187a2a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4ee2345da namespace: kserve-ci-e2e-test resourceVersion: "20872" uid: 86bbf30e-088b-4152-b40c-39bc50b86d46 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4ef8b13a2 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: c9f30ea7-7ad2-4849-9857-029b0e62e6c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4f25db7e1 namespace: kserve-ci-e2e-test resourceVersion: "20875" uid: ef2bfb33-d954-4510-a129-dc3f17d68ac4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4f2cf6226 namespace: kserve-ci-e2e-test resourceVersion: "20876" uid: 2af7603a-dcb8-4631-83ad-3130b2e3f5b9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4f2d9cd77 namespace: kserve-ci-e2e-test resourceVersion: "20877" uid: 1c8947a3-89c9-4078-83d3-5ada6e1c3ce7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4f56eb7d0 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: c18a5e02-9693-4605-88d0-6ca9c2300b51 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:47:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:26Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e4f5e142f6 namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: c52c87d7-d811-435c-936b-188b8a606d1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:47:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:48:03Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:47:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:03Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650e5415cee71 namespace: kserve-ci-e2e-test resourceVersion: "21211" uid: 9d76509b-df36-4c60-ba13-5b2fb1349735 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650f75bef6452 namespace: kserve-ci-e2e-test resourceVersion: "21841" uid: cecc08e8-9f7f-4a0b-a052-728e82e94175 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: c70de991-ec99-4f4a-a9af-44a84b89704b kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-2a946-predictor-85b9467f54-9t9hc.18c650f75befcb45 namespace: kserve-ci-e2e-test resourceVersion: "21845" uid: eaad4d3d-99f4-4a68-ac32-bef8096db028 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-2a946-predictor-85b9467f54 namespace: kserve-ci-e2e-test resourceVersion: "20815" uid: 869432d0-81e1-4bd0-8dd8-f7d8fe7c6cc3 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Created pod: error-404-isvc-2a946-predictor-85b9467f54-9t9hc' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946-predictor-85b9467f54.18c650e4b0392141 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: e7097d5d-c69e-41e0-8b3a-91d2dd0100ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20813" uid: 434b65c0-1a1b-4870-99c8-f46246d181e6 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Scaled up replica set error-404-isvc-2a946-predictor-85b9467f54 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946-predictor.18c650e4ae176923 namespace: kserve-ci-e2e-test resourceVersion: "20816" uid: 97109d47-7972-48a7-a049-77bb3500e813 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 541a7cee-e2f3-4ca8-aa40-2c4baeef0209 kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: error-404-isvc-2a946-predictor.18c650e82c969d34 namespace: kserve-ci-e2e-test resourceVersion: "21290" uid: 30cce47b-2c3e-475c-9ef3-2c2e75c3645e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20814" uid: 541a7cee-e2f3-4ca8-aa40-2c4baeef0209 kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: error-404-isvc-2a946-predictor.18c650e82d01941c namespace: kserve-ci-e2e-test resourceVersion: "21291" uid: 6fee8c2a-dec7-4deb-9083-5853a33b33aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 541a7cee-e2f3-4ca8-aa40-2c4baeef0209 kind: Event lastTimestamp: "2026-07-28T01:48:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:25Z" name: error-404-isvc-2a946-predictor.18c650f2a90f818e namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: fa659b7a-a5cd-4064-be47-97ec3de53a20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "21005" uid: 541a7cee-e2f3-4ca8-aa40-2c4baeef0209 kind: Event lastTimestamp: "2026-07-28T01:48:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:25Z" name: error-404-isvc-2a946-predictor.18c650f2a9567988 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 04b0078c-4fa5-457f-b308-81f77c1a5fe1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 444aa36d-8421-4b2e-95e8-92ee272b77d7 kind: Event lastTimestamp: "2026-07-28T01:48:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:13Z" name: error-404-isvc-2a946.18c650e4ae1fcbb1 namespace: kserve-ci-e2e-test resourceVersion: "21337" uid: 1a236793-33c9-4b8b-9b81-94434dee99c9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 444aa36d-8421-4b2e-95e8-92ee272b77d7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Failed to update status for InferenceService "error-404-isvc-2a946": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2a946": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946.18c650e4aed19fc5 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: db569832-6f78-4604-a3af-fe659ca1240f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20796" uid: 444aa36d-8421-4b2e-95e8-92ee272b77d7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-2a946": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-2a946.18c650e4aed1e84b namespace: kserve-ci-e2e-test resourceVersion: "20821" uid: b3e4884a-541b-4cd7-ad64-b022aa80023e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 444aa36d-8421-4b2e-95e8-92ee272b77d7 kind: Event lastTimestamp: "2026-07-28T01:48:13Z" message: InferenceService [error-404-isvc-2a946] is Ready metadata: creationTimestamp: "2026-07-28T01:48:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:13Z" name: error-404-isvc-2a946.18c650effc78d8e9 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: ad41846e-b065-4253-bcbc-b7ee263fa8ca reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:37:30.825975Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-33843-predictor-77ccc76b96-66hdc to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a3e918d67 namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 2f25e48c-e47b-4934-9ec7-8a903dbbec23 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-33843-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a4a60b180 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: d51a4e56-de49-401a-ba94-2ade33ee4323 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a7bab96db namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 5cae0d2c-a87a-46bb-89c2-e02cd47ef33d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a7cefc48b namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: e8859948-9119-458d-af55-a8d674535e02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a7fa18aad namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: a53fff6b-deba-4fa7-ae76-f8caca771a46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a80030478 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: c88daf91-1483-4541-a133-92c68f9d4ce9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a800d4ef1 namespace: kserve-ci-e2e-test resourceVersion: "16030" uid: 562d56b4-c603-48f3-9827-78fbe5ab6418 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a835e7ccb namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 2a1e8cf3-2c9d-4e8b-94e1-c85dca03aa93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a83bde0c1 namespace: kserve-ci-e2e-test resourceVersion: "16034" uid: 27cc0eab-0045-4ac3-b240-8d09112bce78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:37:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:46:58Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:37:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:58Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c6505a92f541ef namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 659bc29f-50e7-4c21-8f98-f053a5483ecd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c650ddc5adb0bd namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 8a2aa17d-b4e1-4da0-9a6a-2342316749d9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:55Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c650ddc5ae87d8 namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 1cfc0f40-ef4a-448c-9576-5728073f2d57 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-33843-predictor-77ccc76b96-66hdc namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: 34158f2a-5b02-4b81-8a01-c3ee8016668b kind: Event lastTimestamp: "2026-07-28T01:46:58Z" message: 'Readiness probe failed: Get "https://10.133.0.30:8643/healthz": dial tcp 10.133.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:58Z" name: error-404-isvc-33843-predictor-77ccc76b96-66hdc.18c650de5b6893f8 namespace: kserve-ci-e2e-test resourceVersion: "20517" uid: f3404e84-0f71-4f99-ad7b-f500c8ddf062 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-33843-predictor-77ccc76b96 namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 9df8792e-a90d-49bd-a5bc-2f89a1fb264a kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'Created pod: error-404-isvc-33843-predictor-77ccc76b96-66hdc' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-33843-predictor-77ccc76b96.18c6505a3e175119 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 5f336fc0-7de5-4404-bad9-ee60658e78a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: fa99f598-3fd9-4a7b-8083-d36ada3f0dca kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Scaled up replica set error-404-isvc-33843-predictor-77ccc76b96 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-33843-predictor.18c6505a3c666f08 namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: f138a969-e627-4c7b-9eee-53b5bbbecc04 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 789ea2a9-d021-444e-b841-cd3f21882d2b kind: Event lastTimestamp: "2026-07-28T01:38:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:15Z" name: error-404-isvc-33843-predictor.18c6505dbb00c53e namespace: kserve-ci-e2e-test resourceVersion: "16633" uid: b996965b-8a24-4329-8fe1-d38ba01e9dbf reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 789ea2a9-d021-444e-b841-cd3f21882d2b kind: Event lastTimestamp: "2026-07-28T01:38:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:15Z" name: error-404-isvc-33843-predictor.18c6505dbb5ec4c4 namespace: kserve-ci-e2e-test resourceVersion: "16634" uid: 01b0ce45-4e6f-492f-94c4-a2ff28c8dbba reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 789ea2a9-d021-444e-b841-cd3f21882d2b kind: Event lastTimestamp: "2026-07-28T01:38:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:30Z" name: error-404-isvc-33843-predictor.18c65068377be132 namespace: kserve-ci-e2e-test resourceVersion: "16794" uid: 042a58ae-671c-4e91-849f-19db3a4d5459 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "16190" uid: 789ea2a9-d021-444e-b841-cd3f21882d2b kind: Event lastTimestamp: "2026-07-28T01:38:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:30Z" name: error-404-isvc-33843-predictor.18c6506837f6acbf namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: fdf61403-71f7-4bc5-9f82-bdfb4ed2984b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: dbbe25ad-4da9-4a7b-be82-ae846c884ddc kind: Event lastTimestamp: "2026-07-28T01:38:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:18Z" name: error-404-isvc-33843.18c6505a3ca22ea4 namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 688b1bd8-6d53-4c89-8941-af94527f53ce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: dbbe25ad-4da9-4a7b-be82-ae846c884ddc kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'Failed to update status for InferenceService "error-404-isvc-33843": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-33843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-33843.18c6505a3d17f70e namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: 35faebe0-7ede-4539-8ca0-b9989df485e2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: dbbe25ad-4da9-4a7b-be82-ae846c884ddc kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-33843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-33843.18c6505a3d183930 namespace: kserve-ci-e2e-test resourceVersion: "15972" uid: f54aa97d-11f7-4db1-b01e-fc9f18f432ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "16669" uid: dbbe25ad-4da9-4a7b-be82-ae846c884ddc kind: Event lastTimestamp: "2026-07-28T01:38:18Z" message: InferenceService [error-404-isvc-33843] is Ready metadata: creationTimestamp: "2026-07-28T01:38:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:18Z" name: error-404-isvc-33843.18c650654e12b1b4 namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: 16793241-adbb-4281-a332-2c959230f369 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:38:10.740203Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16546" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3d51c-predictor-56c57c964b-rtd7l to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:38:10Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c6506389a44fe2 namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 3d19a5a8-5585-4289-87dc-5d15829af78b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063a2ee851e namespace: kserve-ci-e2e-test resourceVersion: "16566" uid: dd71a880-2b1e-4f00-9134-66d13b238425 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063a461a5e7 namespace: kserve-ci-e2e-test resourceVersion: "16568" uid: 61b48379-f5f9-4d22-b4c2-b10650d82d86 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063a733cb15 namespace: kserve-ci-e2e-test resourceVersion: "16571" uid: bb40bff8-910d-4504-a46a-b9c00597beba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063a7a628a6 namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: b210a730-4d80-4845-89ad-73213e659a6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063a7afe3ed namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: 3b6b589a-17d4-4224-9132-2877412f6d12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063ab9943c3 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: cb594905-50bd-4474-aa84-dc3c17bae0d4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063ac0b9b41 namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 9e707ec2-8252-4a70-a24f-76e9fe35e102 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:47:28Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:28Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c65063ee18d5f1 namespace: kserve-ci-e2e-test resourceVersion: "20899" uid: c5654c5b-39d8-4deb-b5e2-1b30ff787581 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c650e4a911b927 namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: b0fab79e-f10b-41dc-961e-2cfb93dfc46a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c650e4a9122aa2 namespace: kserve-ci-e2e-test resourceVersion: "20794" uid: 5c428735-182f-4bf2-a310-80c3127f6f46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: 635abc6b-b5be-420a-9cca-55974f2e12f0 kind: Event lastTimestamp: "2026-07-28T01:47:28Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:47:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:28Z" name: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l.18c650e5627ade5e namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: aba0622a-800e-4319-8b72-084c6e8c97f1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3d51c-predictor-56c57c964b namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: d47673a7-245b-470e-9bf3-f3a7ee3cb690 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: 'Created pod: error-404-isvc-3d51c-predictor-56c57c964b-rtd7l' metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:10Z" name: error-404-isvc-3d51c-predictor-56c57c964b.18c65063892c4e7e namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 2f40f368-6b5d-49e1-b755-42798955b544 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16533" uid: 8ea07e36-6f00-44a9-9d51-3bad1e8a37d2 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Scaled up replica set error-404-isvc-3d51c-predictor-56c57c964b from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:10Z" name: error-404-isvc-3d51c-predictor.18c65063871db6d2 namespace: kserve-ci-e2e-test resourceVersion: "16539" uid: a362f213-1e07-49f0-9ddb-c5bf3fd2232f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 07b04fec-9b5c-47da-8c62-7d7394498bae kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: error-404-isvc-3d51c-predictor.18c6506706af7ced namespace: kserve-ci-e2e-test resourceVersion: "16872" uid: 92387057-90fa-4197-b78b-9cafd2c5bf59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: 07b04fec-9b5c-47da-8c62-7d7394498bae kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: error-404-isvc-3d51c-predictor.18c65067079ae3e8 namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: af171a5f-6906-4c6c-99ba-b5c77a2500e0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 07b04fec-9b5c-47da-8c62-7d7394498bae kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: error-404-isvc-3d51c-predictor.18c6506e04d082a2 namespace: kserve-ci-e2e-test resourceVersion: "17181" uid: 9d53be9e-141f-48e6-b799-e604e61745c3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: 07b04fec-9b5c-47da-8c62-7d7394498bae kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: error-404-isvc-3d51c-predictor.18c6506e0533699e namespace: kserve-ci-e2e-test resourceVersion: "17182" uid: d2c4e0a9-fa04-4463-a062-d31b5fd0d678 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 80dabe33-67ce-483c-a8d1-62e236165e14 kind: Event lastTimestamp: "2026-07-28T01:38:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:58Z" name: error-404-isvc-3d51c.18c6506387228582 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 9e15f88b-f757-4f11-b033-dcf1dbdad4d1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 80dabe33-67ce-483c-a8d1-62e236165e14 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-3d51c": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d51c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:10Z" name: error-404-isvc-3d51c.18c6506387b97f77 namespace: kserve-ci-e2e-test resourceVersion: "16543" uid: 4f15c6c9-01b0-4cfb-be98-825c05e83e9f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "16508" uid: 80dabe33-67ce-483c-a8d1-62e236165e14 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-3d51c": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:10Z" name: error-404-isvc-3d51c.18c6506387b9bd93 namespace: kserve-ci-e2e-test resourceVersion: "16544" uid: 4c73abc5-9f43-43ee-a7f6-638366cd44e8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: 80dabe33-67ce-483c-a8d1-62e236165e14 kind: Event lastTimestamp: "2026-07-28T01:38:58Z" message: InferenceService [error-404-isvc-3d51c] is Ready metadata: creationTimestamp: "2026-07-28T01:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:58Z" name: error-404-isvc-3d51c.18c6506ea9758737 namespace: kserve-ci-e2e-test resourceVersion: "17056" uid: bb5294d1-9ba5-42be-9b25-cdf38711d3c8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:15.986986Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f075295678 namespace: kserve-ci-e2e-test resourceVersion: "21447" uid: a96a5df6-f4ac-45f3-ae48-5954e94f9cab reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-5a4a0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:16Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0846841f0 namespace: kserve-ci-e2e-test resourceVersion: "21462" uid: 222946ec-ccf6-4043-bc86-718fc36aed5f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21452" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0b26362b5 namespace: kserve-ci-e2e-test resourceVersion: "21475" uid: 01633d6e-4f60-43be-afc9-c2e5477fa412 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0b42db9ea namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 73c9c45a-8044-4659-9ae7-3f749cfc4169 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0b7224155 namespace: kserve-ci-e2e-test resourceVersion: "21483" uid: 010c3e50-9f76-42a6-8c64-baae31e64256 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0b78fdac6 namespace: kserve-ci-e2e-test resourceVersion: "21484" uid: f0072cea-93a9-47e9-a7c4-20689fc3d38c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0b7999e3b namespace: kserve-ci-e2e-test resourceVersion: "21485" uid: 00b75c23-0d82-4152-a32c-01acd9c7113b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0ba9643e2 namespace: kserve-ci-e2e-test resourceVersion: "21486" uid: 65bd4842-85b0-46c6-85cc-e361ed863beb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f0bb05e827 namespace: kserve-ci-e2e-test resourceVersion: "21487" uid: 58d48285-33ab-41c8-b86c-05d5c741bc59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:48:55Z" message: 'Readiness probe failed: dial tcp 10.133.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:55Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c650f12f356402 namespace: kserve-ci-e2e-test resourceVersion: "22007" uid: 68b1260c-cdca-4e28-9a56-d7924fc12ae1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c65173ff0acaa4 namespace: kserve-ci-e2e-test resourceVersion: "26159" uid: 9e480ec2-5f8a-4a47-90eb-9f5ad696e72d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf namespace: kserve-ci-e2e-test resourceVersion: "21444" uid: 195c4c17-1b51-4793-acd9-867ed85d86e5 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf.18c65173ff0b4748 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: 45718ceb-c384-4b79-9a5c-fe84c6128d3c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-5a4a0-predictor-7b676c84f6 namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 28268423-aeb1-4d94-a23b-447a5a252e93 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: 'Created pod: error-404-isvc-5a4a0-predictor-7b676c84f6-hmdgf' metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-5a4a0-predictor-7b676c84f6.18c650f074c7e927 namespace: kserve-ci-e2e-test resourceVersion: "21445" uid: 52fc3fa1-85dc-4a9c-ac95-b53171d266ff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: d289ae42-074e-4759-9e88-5562aa635c84 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Scaled up replica set error-404-isvc-5a4a0-predictor-7b676c84f6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-5a4a0-predictor.18c650f0730b76d2 namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: b14beac0-39f8-4eee-8c5c-d06a7ccb81bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: c13277a8-7a13-4238-b27a-3e29ac025682 kind: Event lastTimestamp: "2026-07-28T01:49:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:46Z" name: error-404-isvc-5a4a0-predictor.18c650f3f16ee836 namespace: kserve-ci-e2e-test resourceVersion: "22487" uid: 5f5a9e6b-92e9-4464-9ae2-c6a300f30549 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: c13277a8-7a13-4238-b27a-3e29ac025682 kind: Event lastTimestamp: "2026-07-28T01:49:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:46Z" name: error-404-isvc-5a4a0-predictor.18c650f3f1cee957 namespace: kserve-ci-e2e-test resourceVersion: "22489" uid: e2900a70-c6eb-4512-85dc-9ea5935e6083 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: c13277a8-7a13-4238-b27a-3e29ac025682 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-5a4a0-predictor.18c650f77085c26d namespace: kserve-ci-e2e-test resourceVersion: "21896" uid: 85bd6f62-8a84-4831-abe0-59154a18b9e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: c13277a8-7a13-4238-b27a-3e29ac025682 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-5a4a0-predictor.18c650f77116ba9c namespace: kserve-ci-e2e-test resourceVersion: "21899" uid: 143840f2-d369-493f-b0aa-8785d85eb52f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 26ed6996-b645-4c61-9305-639d872f2e54 kind: Event lastTimestamp: "2026-07-28T01:49:05Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:05Z" name: error-404-isvc-5a4a0.18c650f0730c4c53 namespace: kserve-ci-e2e-test resourceVersion: "22098" uid: d168efbd-4e7c-49a5-94c0-3e4667bd2902 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 26ed6996-b645-4c61-9305-639d872f2e54 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: 'Failed to update status for InferenceService "error-404-isvc-5a4a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5a4a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-5a4a0.18c650f073df0400 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: abab2c8c-b877-47c2-bd16-bfcfec5f1b58 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "21417" uid: 26ed6996-b645-4c61-9305-639d872f2e54 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-5a4a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:15Z" name: error-404-isvc-5a4a0.18c650f073df5595 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 72dd1cdb-a5b4-4bf2-b2c4-612d1b5273a3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:05Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 26ed6996-b645-4c61-9305-639d872f2e54 kind: Event lastTimestamp: "2026-07-28T01:49:05Z" message: InferenceService [error-404-isvc-5a4a0] is Ready metadata: creationTimestamp: "2026-07-28T01:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:05Z" name: error-404-isvc-5a4a0.18c650fbea50b23e namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: a41e4c4f-2a67-4885-9ad7-917962290a84 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:58:10.462119Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26517" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517adea3fc43 namespace: kserve-ci-e2e-test resourceVersion: "26521" uid: 9de6a2e6-727a-4d2b-b0a9-4bf422b955ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26528" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Add eth0 [10.133.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517af83e832c namespace: kserve-ci-e2e-test resourceVersion: "26550" uid: 0ad6f82e-f316-47d3-bd1d-4c4f4301b86f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517af9a3fb50 namespace: kserve-ci-e2e-test resourceVersion: "26552" uid: cc56062e-8e9a-4cc1-ae5e-4b32df83052d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517afd5fd53f namespace: kserve-ci-e2e-test resourceVersion: "26553" uid: 3a0681bb-8f45-4e77-b970-e156f0ef35bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517afdd20055 namespace: kserve-ci-e2e-test resourceVersion: "26554" uid: 6ad70586-efa7-4f11-8ead-c023110d0c85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517afddba70e namespace: kserve-ci-e2e-test resourceVersion: "26555" uid: 48182d39-506f-4303-b1b3-ece2d5da80f4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517b00957059 namespace: kserve-ci-e2e-test resourceVersion: "26556" uid: 785c87cc-b684-4610-b6d6-f330e1d008f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517b01091ddb namespace: kserve-ci-e2e-test resourceVersion: "26557" uid: 9f8912fc-ed21-4ecd-b49d-98ca1b1a0c47 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T01:58:48Z" message: 'Readiness probe failed: dial tcp 10.133.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:48Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6517b63547b3e namespace: kserve-ci-e2e-test resourceVersion: "26947" uid: bcde9548-665d-40ff-b0a0-dd0e3180c84c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T02:15:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T02:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:39Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6526f2cdf7fe7 namespace: kserve-ci-e2e-test resourceVersion: "34305" uid: ea570198-00d1-4955-870d-414ad1dda839 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6 namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c30bf6ba-be23-4ecf-8f1c-65676592a689 kind: Event lastTimestamp: "2026-07-28T02:15:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T02:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:39Z" name: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6.18c6526f2cdff5e9 namespace: kserve-ci-e2e-test resourceVersion: "34306" uid: bb433eb7-5c31-4731-9c31-6ff93dcf2231 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-704b9-predictor-c58dcf5fd namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: 86bff583-3eb2-4014-bdc9-39671359590e kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'Created pod: error-404-isvc-704b9-predictor-c58dcf5fd-fxxl6' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor-c58dcf5fd.18c6517ade3eab2d namespace: kserve-ci-e2e-test resourceVersion: "26518" uid: 8cca2440-cc54-49e3-9f21-9400afddee37 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: e9fe90e7-ab71-4539-9400-e3f50b0849ba kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Scaled up replica set error-404-isvc-704b9-predictor-c58dcf5fd from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9-predictor.18c6517adc83e023 namespace: kserve-ci-e2e-test resourceVersion: "26512" uid: befb2813-07cf-4497-8ce7-fe40b1f83a0e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: 0b407b59-d461-4b6a-b689-3aaef6a9869c kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: error-404-isvc-704b9-predictor.18c6517e5c619006 namespace: kserve-ci-e2e-test resourceVersion: "26844" uid: 61b0089e-f8f1-4848-8916-401664137d30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: 0b407b59-d461-4b6a-b689-3aaef6a9869c kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: error-404-isvc-704b9-predictor.18c6517e5d8d066c namespace: kserve-ci-e2e-test resourceVersion: "26845" uid: 2fdfe755-2649-4aea-81a0-4fe6861c17af reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: 0b407b59-d461-4b6a-b689-3aaef6a9869c kind: Event lastTimestamp: "2026-07-28T01:59:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:25Z" name: error-404-isvc-704b9-predictor.18c651855a8f5dd0 namespace: kserve-ci-e2e-test resourceVersion: "27439" uid: cf516bc4-31c7-4cec-8b12-14a1c87f4cd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:58:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26700" uid: 0b407b59-d461-4b6a-b689-3aaef6a9869c kind: Event lastTimestamp: "2026-07-28T01:59:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:25Z" name: error-404-isvc-704b9-predictor.18c651855af87b3a namespace: kserve-ci-e2e-test resourceVersion: "27440" uid: 1bbabf02-82da-4247-8cd0-45a89396a870 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 1d631ebb-34b4-4bb4-a2ec-eebba038ea20 kind: Event lastTimestamp: "2026-07-28T01:58:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:58Z" name: error-404-isvc-704b9.18c6517adc89b3fb namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: 1acfd783-64eb-495e-8151-70b8e7105098 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 1d631ebb-34b4-4bb4-a2ec-eebba038ea20 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'Failed to update status for InferenceService "error-404-isvc-704b9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-704b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9.18c6517add05af15 namespace: kserve-ci-e2e-test resourceVersion: "26514" uid: 8c25f202-2418-44c0-9b23-4cf8b5f77f8e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "26490" uid: 1d631ebb-34b4-4bb4-a2ec-eebba038ea20 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-704b9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-704b9.18c6517add05f876 namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: d13bbdb3-a1ec-4ede-9427-b71f6eee2093 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27179" uid: 1d631ebb-34b4-4bb4-a2ec-eebba038ea20 kind: Event lastTimestamp: "2026-07-28T01:58:58Z" message: InferenceService [error-404-isvc-704b9] is Ready metadata: creationTimestamp: "2026-07-28T01:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:58Z" name: error-404-isvc-704b9.18c651861ef27a2f namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: e1338345-95cd-4b6e-8eba-ed480b9ad8a8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:58:51.140190Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27054" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7876a-predictor-7fd999b76-6ws2h to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65184573e0364 namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 2432aac7-5940-4988-941b-8e7863dbf0c3 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Add eth0 [10.133.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c6518470bbee02 namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 90999007-1a47-43a7-ac1a-fbc4cdd541ba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c6518471fec32b namespace: kserve-ci-e2e-test resourceVersion: "27081" uid: 4d432029-882c-47b2-ab9e-12c238ac98b0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c6518474a0e994 namespace: kserve-ci-e2e-test resourceVersion: "27082" uid: a687f408-8cc9-4f32-98d1-59d552369774 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65184750d3c32 namespace: kserve-ci-e2e-test resourceVersion: "27083" uid: 75bfc44d-bad8-4db1-b039-279f5d5674a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c6518475166faa namespace: kserve-ci-e2e-test resourceVersion: "27084" uid: 19cc2ef3-d095-495e-b5ca-9a40b95c7b1a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c6518477e87c4d namespace: kserve-ci-e2e-test resourceVersion: "27086" uid: be6a643f-95d5-44ff-beca-eb2e9da98fed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c651847855eb21 namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: bcdf926f-dd06-4412-b008-6923135c2f3c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T02:08:08Z" message: 'Readiness probe failed: dial tcp 10.133.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:08Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65184be312946 namespace: kserve-ci-e2e-test resourceVersion: "31185" uid: 8a1c3021-3211-4b00-a53f-09db473e1a1b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T02:08:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T02:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:05Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65205824c2a01 namespace: kserve-ci-e2e-test resourceVersion: "31161" uid: cd28f7af-1c3a-4cc3-a7b7-2bf122045c0c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T02:08:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T02:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:05Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65205824c7eb0 namespace: kserve-ci-e2e-test resourceVersion: "31159" uid: bfbe4261-e8bb-4269-b4dd-3c946b2d5b2f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h namespace: kserve-ci-e2e-test resourceVersion: "27056" uid: c8f296f4-a37f-475d-8b5a-cf834952cd3c kind: Event lastTimestamp: "2026-07-28T02:08:08Z" message: 'Readiness probe failed: Get "https://10.133.0.40:8643/healthz": dial tcp 10.133.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T02:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:08Z" name: error-404-isvc-7876a-predictor-7fd999b76-6ws2h.18c65206329e5876 namespace: kserve-ci-e2e-test resourceVersion: "31184" uid: 3ceb30ea-3ab6-46ac-93e3-169de030fece reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7876a-predictor-7fd999b76 namespace: kserve-ci-e2e-test resourceVersion: "27047" uid: 23a2cd1c-a79e-40bd-ab63-5b4ca817fed2 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Created pod: error-404-isvc-7876a-predictor-7fd999b76-6ws2h' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor-7fd999b76.18c6518456d7c71b namespace: kserve-ci-e2e-test resourceVersion: "27055" uid: 80e47b4a-ef63-4d33-b5a8-2def74c83004 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27045" uid: 96e180a4-e300-48fb-8202-d64e5f657062 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Scaled up replica set error-404-isvc-7876a-predictor-7fd999b76 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a-predictor.18c65184551d7713 namespace: kserve-ci-e2e-test resourceVersion: "27048" uid: a5742e15-c276-4ce2-8e68-413779474509 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: b3958d32-9633-4ae9-81da-b747de43b455 kind: Event lastTimestamp: "2026-07-28T01:59:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:06Z" name: error-404-isvc-7876a-predictor.18c65187d3a3257a namespace: kserve-ci-e2e-test resourceVersion: "27242" uid: 2b3ab8b4-73e8-4655-911f-40572e1a1975 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27046" uid: b3958d32-9633-4ae9-81da-b747de43b455 kind: Event lastTimestamp: "2026-07-28T01:59:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:06Z" name: error-404-isvc-7876a-predictor.18c65187d3ed35f9 namespace: kserve-ci-e2e-test resourceVersion: "27244" uid: 3eea25a2-6d1e-4660-a0ae-883c9aa2bc97 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: b3958d32-9633-4ae9-81da-b747de43b455 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: error-404-isvc-7876a-predictor.18c6518b52a33285 namespace: kserve-ci-e2e-test resourceVersion: "27805" uid: d62f0a2f-bafb-40f1-9ed0-dbefa6b12c8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27243" uid: b3958d32-9633-4ae9-81da-b747de43b455 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: error-404-isvc-7876a-predictor.18c6518b5320070f namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: e7b0f08a-e3bc-4130-8cd5-a769d68e7a79 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: a5174b17-103b-4f0e-94c6-d2e60be548ee kind: Event lastTimestamp: "2026-07-28T01:59:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:39Z" name: error-404-isvc-7876a.18c65184552491c0 namespace: kserve-ci-e2e-test resourceVersion: "27560" uid: e1c6c42e-73dc-457d-864a-ff511a22622c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: a5174b17-103b-4f0e-94c6-d2e60be548ee kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-7876a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7876a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a.18c651845590feb0 namespace: kserve-ci-e2e-test resourceVersion: "27051" uid: 812d4712-2969-482a-bea1-583b477fb8c1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "27029" uid: a5174b17-103b-4f0e-94c6-d2e60be548ee kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7876a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:51Z" name: error-404-isvc-7876a.18c6518455914a43 namespace: kserve-ci-e2e-test resourceVersion: "27052" uid: 2553fac0-3ed8-49d7-8498-6ab4e9d1e05e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "27553" uid: a5174b17-103b-4f0e-94c6-d2e60be548ee kind: Event lastTimestamp: "2026-07-28T01:59:38Z" message: InferenceService [error-404-isvc-7876a] is Ready metadata: creationTimestamp: "2026-07-28T01:59:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:38Z" name: error-404-isvc-7876a.18c6518f7a02e794 namespace: kserve-ci-e2e-test resourceVersion: "27554" uid: 90bb6228-45dc-49ab-a84f-0e7647ca54be reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:45.756135Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21871" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-920f8-predictor-674db79dcb-hgrcz to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f7638a8257 namespace: kserve-ci-e2e-test resourceVersion: "21875" uid: f6248478-8c1e-4cf9-a237-f4e4189b82e9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21880" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f77d111e11 namespace: kserve-ci-e2e-test resourceVersion: "21902" uid: 0df3e712-d22c-45bd-94a6-52ef746948d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f77e9b2187 namespace: kserve-ci-e2e-test resourceVersion: "21904" uid: 477a2eb1-2c32-4c7f-ad92-ded85e52d502 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f781225eea namespace: kserve-ci-e2e-test resourceVersion: "21905" uid: 2ee150e9-a60b-4573-b796-52a5b4461904 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f781922b82 namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: ea899ce1-c032-440e-bbde-b2a5ccf622dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f7819bd741 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: f9dd3ded-6775-4629-b813-a183608a495f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f7847b860c namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: e9700361-481f-493a-bf2b-292b810bb790 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f784f19c31 namespace: kserve-ci-e2e-test resourceVersion: "21909" uid: bdd5203d-ec9d-43e7-8502-1fb7e6cc040b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:58:13Z" message: 'Readiness probe failed: dial tcp 10.133.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:13Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c650f7bc063d94 namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: b1686990-e26d-475b-bfba-c2b11e617603 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c6517ad7cdff01 namespace: kserve-ci-e2e-test resourceVersion: "26491" uid: f269f287-dda9-4e4a-9088-a959c87e5741 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c6517ad7ce6e23 namespace: kserve-ci-e2e-test resourceVersion: "26495" uid: eea0be01-21bb-4bfc-8845-a11efbefca7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz namespace: kserve-ci-e2e-test resourceVersion: "21873" uid: 6d3a5956-07f5-42d9-9961-606f2732abf6 kind: Event lastTimestamp: "2026-07-28T01:58:13Z" message: 'Readiness probe failed: Get "https://10.133.0.37:8643/healthz": dial tcp 10.133.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:13Z" name: error-404-isvc-920f8-predictor-674db79dcb-hgrcz.18c6517b848196ef namespace: kserve-ci-e2e-test resourceVersion: "26596" uid: f2b8e122-ad41-48c4-9364-4c47f6d6b495 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-920f8-predictor-674db79dcb namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: 8ae60ad4-2940-4f9c-8078-a5a3df9a34c4 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'Created pod: error-404-isvc-920f8-predictor-674db79dcb-hgrcz' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-920f8-predictor-674db79dcb.18c650f7631e8899 namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: 5146f9ca-9877-437f-aa28-3c4804e8f9b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: d740dc12-1d28-41c5-b4fe-e9c8ca946e19 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Scaled up replica set error-404-isvc-920f8-predictor-674db79dcb from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-920f8-predictor.18c650f760b199c9 namespace: kserve-ci-e2e-test resourceVersion: "21864" uid: 63244e11-da25-48c9-b23e-1e5f93421fdf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 1182bd3f-908d-40fe-b1e3-7fa5712efccd kind: Event lastTimestamp: "2026-07-28T01:49:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:30Z" name: error-404-isvc-920f8-predictor.18c650fadf13ed84 namespace: kserve-ci-e2e-test resourceVersion: "22340" uid: ea3a40ab-dd3d-4d0b-863d-c80f7951ea64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: 1182bd3f-908d-40fe-b1e3-7fa5712efccd kind: Event lastTimestamp: "2026-07-28T01:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:30Z" name: error-404-isvc-920f8-predictor.18c650fadf8ada7e namespace: kserve-ci-e2e-test resourceVersion: "22341" uid: 3fbeb89c-4673-4c08-a1c0-acceb8573388 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 1182bd3f-908d-40fe-b1e3-7fa5712efccd kind: Event lastTimestamp: "2026-07-28T01:50:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:15Z" name: error-404-isvc-920f8-predictor.18c651055b82d3b4 namespace: kserve-ci-e2e-test resourceVersion: "22747" uid: 68266cdf-55f3-4d6a-9da2-6d594ca7a133 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22047" uid: 1182bd3f-908d-40fe-b1e3-7fa5712efccd kind: Event lastTimestamp: "2026-07-28T01:50:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:15Z" name: error-404-isvc-920f8-predictor.18c651055be29d16 namespace: kserve-ci-e2e-test resourceVersion: "22748" uid: 45a96a99-2769-4695-90db-1182afeaa7d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 9900bd5e-6668-491f-98c3-05c411d3f6cc kind: Event lastTimestamp: "2026-07-28T01:49:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:33Z" name: error-404-isvc-920f8.18c650f760bc16a4 namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 70eed09f-9a89-46f6-a331-a65c722cf002 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 9900bd5e-6668-491f-98c3-05c411d3f6cc kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-920f8": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-920f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-920f8.18c650f761f01823 namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: ed9971a7-9a4c-4b21-8939-0039e83f6109 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "21840" uid: 9900bd5e-6668-491f-98c3-05c411d3f6cc kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-920f8": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:45Z" name: error-404-isvc-920f8.18c650f761f05cfd namespace: kserve-ci-e2e-test resourceVersion: "21870" uid: 918ecbe4-dc4a-451c-a7f7-a6af7a7147a9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: 9900bd5e-6668-491f-98c3-05c411d3f6cc kind: Event lastTimestamp: "2026-07-28T01:49:33Z" message: InferenceService [error-404-isvc-920f8] is Ready metadata: creationTimestamp: "2026-07-28T01:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:33Z" name: error-404-isvc-920f8.18c651027732d97a namespace: kserve-ci-e2e-test resourceVersion: "22375" uid: cbe9fa9a-3f9c-4eb7-8fdd-d915b005a795 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:57:40.872327Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26132" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c65173faf38bbe namespace: kserve-ci-e2e-test resourceVersion: "26139" uid: 8cce2317-3940-4b1d-9f9c-ec03ee3ea178 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-bc47b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c65174056821d2 namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: 5ef72385-866b-4499-a9da-d36e1113f24a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26145" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651743829aab8 namespace: kserve-ci-e2e-test resourceVersion: "26191" uid: 5a8a1f85-17f5-4b03-9777-9064b1f2d9e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c65174397aad10 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 565cb893-a1a3-41e9-86f1-cf593bc45585 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651743d2cae94 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 3d46ddc7-f278-48ad-bc89-a675fe304ff6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651743d9fce0e namespace: kserve-ci-e2e-test resourceVersion: "26195" uid: 783b6303-0692-4000-812a-d61db632f294 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651743daaaef6 namespace: kserve-ci-e2e-test resourceVersion: "26196" uid: 233f577b-7f35-46da-af12-1143f256a786 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:42Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c6517440c6b11a namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 4d61182a-6f8c-4b72-ae9f-6b2d1a6bae46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:57:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:42Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c65174413fa277 namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: c2b1e8dd-8346-4634-8182-8305e777b158 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:57:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:58:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:57:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:18Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651745f03c758 namespace: kserve-ci-e2e-test resourceVersion: "26645" uid: 1aac7bf1-389f-43e8-bbde-ae8ed284c2ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:58:50Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:50Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651844c2139a7 namespace: kserve-ci-e2e-test resourceVersion: "26995" uid: cf7d66aa-57d8-4d1b-8785-d8d744ceb41d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:58:50Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:50Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c651844c21aeaf namespace: kserve-ci-e2e-test resourceVersion: "26997" uid: 379dd68e-3088-4440-9593-b3cc8225e755 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 7bdcd365-66da-4351-a436-7ad7647d133b kind: Event lastTimestamp: "2026-07-28T01:58:53Z" message: 'Readiness probe failed: Get "https://10.133.0.38:8643/healthz": dial tcp 10.133.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:53Z" name: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66.18c65184e727bf9e namespace: kserve-ci-e2e-test resourceVersion: "27123" uid: 979eb704-c6ff-490f-8403-18579d4f5d1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-bc47b-predictor-c6d4595c4 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: c398b719-a366-4d1a-b4ca-8113fd09eb96 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'Created pod: error-404-isvc-bc47b-predictor-c6d4595c4-zrw66' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-bc47b-predictor-c6d4595c4.18c65173fa3177dd namespace: kserve-ci-e2e-test resourceVersion: "26135" uid: c04b8c63-61f9-4c02-9d83-c78212e35399 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: aef1dbb6-833b-40ea-9dcb-6bf5b474fd7e kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Scaled up replica set error-404-isvc-bc47b-predictor-c6d4595c4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-bc47b-predictor.18c65173f81221fc namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: 2ba64752-86bd-4643-8f75-b86c4cb7fa28 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:57:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 4adf13f0-8ae0-4344-93b5-14b20650f433 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-bc47b-predictor.18c651777678c862 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: a72fe35a-a1bc-4a56-baae-6881a3d1e354 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:57:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 4adf13f0-8ae0-4344-93b5-14b20650f433 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: error-404-isvc-bc47b-predictor.18c651777703264a namespace: kserve-ci-e2e-test resourceVersion: "26548" uid: baddc19a-60d9-4871-bf66-328cff196cc7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: 4adf13f0-8ae0-4344-93b5-14b20650f433 kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: error-404-isvc-bc47b-predictor.18c6517e749cc641 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 13123c56-4628-4a74-9f36-34ecb3a4dbc6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26322" uid: 4adf13f0-8ae0-4344-93b5-14b20650f433 kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: error-404-isvc-bc47b-predictor.18c6517e74f83d2a namespace: kserve-ci-e2e-test resourceVersion: "26880" uid: b5baa760-ca58-4a49-a5a8-8cd1a5615c04 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: 3dff8400-b8db-47af-8bd9-a2efa0c8ef1d kind: Event lastTimestamp: "2026-07-28T01:58:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:28Z" name: error-404-isvc-bc47b.18c65173f8185bc1 namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: d1f322ee-cbc0-4ed3-9e22-a07adf66cd24 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: 3dff8400-b8db-47af-8bd9-a2efa0c8ef1d kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'Failed to update status for InferenceService "error-404-isvc-bc47b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bc47b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-bc47b.18c65173f8da2bcd namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: ccc54e55-9149-404f-876a-d1a5daf072a4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26106" uid: 3dff8400-b8db-47af-8bd9-a2efa0c8ef1d kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-bc47b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:57:40Z" name: error-404-isvc-bc47b.18c65173f8da7c30 namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: ee0a53d6-ffa0-49db-bb12-0f9f5e2d40bd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26740" uid: 3dff8400-b8db-47af-8bd9-a2efa0c8ef1d kind: Event lastTimestamp: "2026-07-28T01:58:28Z" message: InferenceService [error-404-isvc-bc47b] is Ready metadata: creationTimestamp: "2026-07-28T01:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:28Z" name: error-404-isvc-bc47b.18c6517f1b101e50 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 884324bf-c6be-4ace-9fcb-5069b6398092 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:35:50.754785Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14875" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c7854-predictor-55858f5444-2l7kd to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:35:50Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65042f1dc9353 namespace: kserve-ci-e2e-test resourceVersion: "14881" uid: 91a48b08-5b0e-4720-a795-f55309572296 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c7854-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:50Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65042fdc8b501 namespace: kserve-ci-e2e-test resourceVersion: "14919" uid: 17ba2ebe-340b-489e-9f5f-a443fd1067d1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:35:51Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c650432f0cf0cc namespace: kserve-ci-e2e-test resourceVersion: "14952" uid: 24d44b2a-0d9b-44f7-9ade-517aeb85bf39 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65043304ad991 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 2d51189e-4ad6-4628-9ab2-cb0239e2727c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:03Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" in 12.079s (12.079s including waiting). Image size: 1333937259 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:03Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65046004557fe namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: 205f633d-86a4-4727-8db3-cf6716bed160 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:03Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65046036e5bd1 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 90aa275f-52ce-4df9-8223-dd0ca37f2ff1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:03Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6504603e0f54a namespace: kserve-ci-e2e-test resourceVersion: "15079" uid: 4d88cd98-09d6-49a2-aaf4-d3cd3dd5e9f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:03Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T01:36:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:03Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6504603ef9595 namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: eea72a56-c889-4893-97d2-b4ef0fee3f73 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:06Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.303s (2.303s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:06Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c650468d3b1e27 namespace: kserve-ci-e2e-test resourceVersion: "15116" uid: 88d797c3-c015-4ea5-889c-f62ec6336afd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:06Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6504690243788 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: 97be2bf2-1f2d-40cb-9ea4-0c05f76a8367 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:36:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:06Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c650469088bb4d namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 28c7546a-16f4-45bd-adf4-78262522eee9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:36:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:37:32Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:36:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:32Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c65046b278bce1 namespace: kserve-ci-e2e-test resourceVersion: "16050" uid: 792e1e5d-e9b8-415d-ada2-07d614ddfa8a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6505a4226ba65 namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: de61cb31-1acf-47cd-9402-6d9ffabc83dd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6505a4227242d namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: 8a3511b2-0b72-4c60-b39d-c4ad64651c7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7854-predictor-55858f5444-2l7kd namespace: kserve-ci-e2e-test resourceVersion: "14879" uid: bcd1b450-ad80-459f-98f7-0eb778b6320a kind: Event lastTimestamp: "2026-07-28T01:37:32Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": dial tcp 10.133.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:37:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:32Z" name: error-404-isvc-c7854-predictor-55858f5444-2l7kd.18c6505ab8ad5ded namespace: kserve-ci-e2e-test resourceVersion: "16049" uid: d3681f23-7c4e-4602-b40f-f413699bd371 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c7854-predictor-55858f5444 namespace: kserve-ci-e2e-test resourceVersion: "14868" uid: 61b82884-5962-4e55-8378-e60d3dbe36ab kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Created pod: error-404-isvc-c7854-predictor-55858f5444-2l7kd' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: error-404-isvc-c7854-predictor-55858f5444.18c65042f1598148 namespace: kserve-ci-e2e-test resourceVersion: "14878" uid: 763281b0-8131-4b05-8480-05a92be2dd3c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14865" uid: 73bb4367-c90b-4ec0-96bd-2002dc32f4f9 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Scaled up replica set error-404-isvc-c7854-predictor-55858f5444 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: error-404-isvc-c7854-predictor.18c65042eec873f0 namespace: kserve-ci-e2e-test resourceVersion: "14869" uid: f5e4edbe-6d21-4c32-a64b-da79d7a164b0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 053100e9-f0fb-42e5-b1ce-d05deeacc024 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: error-404-isvc-c7854-predictor.18c650466d531366 namespace: kserve-ci-e2e-test resourceVersion: "15395" uid: 13bfdf9b-65be-4038-9e90-72c5b8d07c26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14867" uid: 053100e9-f0fb-42e5-b1ce-d05deeacc024 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: error-404-isvc-c7854-predictor.18c650466d987ee1 namespace: kserve-ci-e2e-test resourceVersion: "15396" uid: aaf4135b-60a7-4290-b062-0ae7a1e3a124 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:36:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 053100e9-f0fb-42e5-b1ce-d05deeacc024 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: error-404-isvc-c7854-predictor.18c65049ec3b4e0b namespace: kserve-ci-e2e-test resourceVersion: "15806" uid: cccf726d-805a-4af3-a425-bc4c5b517523 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:36:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 053100e9-f0fb-42e5-b1ce-d05deeacc024 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: error-404-isvc-c7854-predictor.18c65049ec7ea8fe namespace: kserve-ci-e2e-test resourceVersion: "15807" uid: b7853e44-5606-4754-ac76-5b2f293cec98 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "14849" uid: e2ffd52d-1836-4322-88d0-c124b60d23f8 kind: Event lastTimestamp: "2026-07-28T01:36:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:36:52Z" name: error-404-isvc-c7854.18c65042eedc556d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: 7df43977-d061-475b-9bda-869b3a006df8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "15547" uid: e2ffd52d-1836-4322-88d0-c124b60d23f8 kind: Event lastTimestamp: "2026-07-28T01:36:52Z" message: InferenceService [error-404-isvc-c7854] is Ready metadata: creationTimestamp: "2026-07-28T01:36:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:36:52Z" name: error-404-isvc-c7854.18c650516dc74866 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 99a15e11-3b74-4df7-8acf-dd5026304613 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:35:50.576672Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14826" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c65042e73e87bb namespace: kserve-ci-e2e-test resourceVersion: "14831" uid: e672e666-0f6d-4294-b223-2c7c0094cd8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c65042f4d2fa39 namespace: kserve-ci-e2e-test resourceVersion: "14896" uid: 443bac39-8793-4356-ab91-184724ccecca reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504331d8d375 namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: bc398f3e-2a31-44bb-b8ed-075a227318a3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c65043339eb8d5 namespace: kserve-ci-e2e-test resourceVersion: "14957" uid: 4ca05890-e0d3-4ee5-b767-c91bf7426988 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c65043386f7479 namespace: kserve-ci-e2e-test resourceVersion: "14958" uid: d6d7e006-df4d-42d4-a9e0-210ea557aa84 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650433a33e8ec namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 928f640b-aa85-4cd5-850a-e234e268f5b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:35:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T01:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:57Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504472e370e7 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: b67b0c0c-13e9-4753-95c7-02976c62c254 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 11.028s (11.028s including waiting). Image size: 1555730357 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650470436c1dc namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: bc35641f-89ed-4682-8830-40200e06b61a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504707acd19e namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 8fad9929-c0d6-4ccf-8f7e-f4cd4777af11 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650470842ec5b namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: 4470792c-1e82-4288-ab4e-df6186067360 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504708a376ef namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: 6373bf1a-998d-4a2f-bc66-ea07a336af14 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.489s (4.489s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c65048143a4291 namespace: kserve-ci-e2e-test resourceVersion: "15178" uid: 7382f364-1cfd-41c5-8400-d3b5d5fa8a53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504819422f1f namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 6aa3afa5-d739-4d74-aae2-fd58ddb06bdc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650481a02aed2 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: cd4975ec-13a3-4215-83a7-bdb1ba582ca6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T01:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:37:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:29Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c6504833207a63 namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: 52d94b8a-380c-4386-a25c-07bedd0cf688 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650637f70149c namespace: kserve-ci-e2e-test resourceVersion: "16499" uid: 26ebbb6d-cdb6-4d13-b563-a2ad0d8dd0ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650637f714308 namespace: kserve-ci-e2e-test resourceVersion: "16501" uid: 758b1c7f-b28d-4699-8245-490b6fa88aa1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf namespace: kserve-ci-e2e-test resourceVersion: "14828" uid: ffa80abb-6239-4e15-b294-cfde80f6becf kind: Event lastTimestamp: "2026-07-28T01:38:14Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:38:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:14Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf.18c650645fbec12f namespace: kserve-ci-e2e-test resourceVersion: "16619" uid: 68e06b0d-169d-4abc-8d91-f1e0a0b077db reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d55f5d589 namespace: kserve-ci-e2e-test resourceVersion: "14817" uid: ca859e03-ce60-461b-b68f-61c112af6000 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d55f5d589-jvccf' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589.18c65042e6b0b028 namespace: kserve-ci-e2e-test resourceVersion: "14827" uid: 2d75c8bf-dbb0-48df-b025-dcee0643d62d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14815" uid: bb67fbef-abe9-4633-a9fd-5a98d11d5bdf kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d55f5d589 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-1-predictor.18c65042e2b6fcb9 namespace: kserve-ci-e2e-test resourceVersion: "14818" uid: 40eaa6e3-4cea-473f-a50b-5bf32e2012e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 812486a5-79e0-4303-ac47-f94c31e81037 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: isvc-sklearn-graph-1-predictor.18c65046612d9757 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: 4618f358-fd03-42b2-b963-b5ccbb923eb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 812486a5-79e0-4303-ac47-f94c31e81037 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: isvc-sklearn-graph-1-predictor.18c650466194d982 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: e9d74c65-6663-4137-b8e6-54f5c84047fd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 812486a5-79e0-4303-ac47-f94c31e81037 kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-sklearn-graph-1-predictor.18c65050dd9d407d namespace: kserve-ci-e2e-test resourceVersion: "16384" uid: 914ee5fe-183b-49c6-83e1-07ff55c4a116 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 812486a5-79e0-4303-ac47-f94c31e81037 kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-sklearn-graph-1-predictor.18c65050de040224 namespace: kserve-ci-e2e-test resourceVersion: "16385" uid: e11e46a5-d8b3-4c9a-9ad2-cc8f437195fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14803" uid: be7ecda0-36a7-4912-82a4-1f49f6f304db kind: Event lastTimestamp: "2026-07-28T01:37:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:39Z" name: isvc-sklearn-graph-1.18c65042e2e80ec0 namespace: kserve-ci-e2e-test resourceVersion: "16133" uid: be80d46c-3ff1-4aa4-a17a-0b6eb0d61bd1 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16131" uid: be7ecda0-36a7-4912-82a4-1f49f6f304db kind: Event lastTimestamp: "2026-07-28T01:37:39Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T01:37:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:39Z" name: isvc-sklearn-graph-1.18c6505c3ef93b26 namespace: kserve-ci-e2e-test resourceVersion: "16132" uid: 734b57c8-ff1f-4393-972a-174e8ed2961b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:35:50.875289Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d44765f95-zc72k to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c65042f90b1d7c namespace: kserve-ci-e2e-test resourceVersion: "14909" uid: 2010ad0a-9fb9-40fa-bc46-2bac88dbb3bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14915" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c65043131743eb namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: c97eabbb-f39e-4c1b-b1f6-5b5fb190154e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c6504314a1077b namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: ec681bf5-8679-4239-89fe-49daf7bb7b85 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650431884848d namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: cce272a8-c0a4-47e4-83fb-1d6277d00adb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c65043192eee24 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 254c6321-d283-498e-889c-8d9881b59611 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:35:57Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T01:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:57Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c65044731d9583 namespace: kserve-ci-e2e-test resourceVersion: "15019" uid: 77bc5834-cdd8-4c0c-99a0-5f31c26b5acc reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 11.07s (11.07s including waiting). Image size: 1555730357 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c6504706f111b1 namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 38d32200-40ec-43c3-baa2-a8d768eba81a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650470acaab53 namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 57730e09-3577-4def-9efb-c86d4dc2a11a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650470baf9f07 namespace: kserve-ci-e2e-test resourceVersion: "15146" uid: e01d554c-2864-4599-8e83-d3cbeb74813e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650470bca42f2 namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 73c44191-6a61-42ba-aea1-f6b44f9231b7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.394s (4.394s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c6504811c0259d namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 1779c7fd-6a71-4288-a0d4-13442d3af392 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c6504815aa1867 namespace: kserve-ci-e2e-test resourceVersion: "15179" uid: 2a51b2fc-d96a-4bcb-a3d3-383ce0b76351 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c6504816af6fbb namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: 9745d213-4af6-4715-b4d4-12008f551ee4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T01:36:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:36:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650486f22ffe0 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 2b71a507-dc46-4165-bbab-73e634cdd10e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650637c07bbf0 namespace: kserve-ci-e2e-test resourceVersion: "16470" uid: 7229131a-2308-4c8a-85ef-d6a0fd02c9e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650637c083174 namespace: kserve-ci-e2e-test resourceVersion: "16471" uid: dfc84716-5da1-4f48-8f5d-c367f5246cb6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: 87fd680b-b995-49c6-ad0d-62c45a833494 kind: Event lastTimestamp: "2026-07-28T01:38:15Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:38:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:15Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k.18c650649ba160d1 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: c2a47559-fda4-4f6b-9e01-1b867c67392b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d44765f95 namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: ed4f1af6-cd2d-4229-829b-48428b4c8493 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d44765f95-zc72k' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-2-predictor-7d44765f95.18c65042f8956ecb namespace: kserve-ci-e2e-test resourceVersion: "14906" uid: 0555422b-2fab-4156-b3b5-8443abcfe266 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14892" uid: b5797c3b-2a16-4abb-a7e7-08a3317a6a0c kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d44765f95 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-sklearn-graph-2-predictor.18c65042f51102ce namespace: kserve-ci-e2e-test resourceVersion: "14898" uid: 9f34d3f1-cbc9-4139-a8c4-c0fc200dcea8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: bb5907fb-dc42-4963-875c-24ec36ca4fcd kind: Event lastTimestamp: "2026-07-28T01:36:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:50Z" name: isvc-sklearn-graph-2-predictor.18c65046737c4f9a namespace: kserve-ci-e2e-test resourceVersion: "15525" uid: 634223b2-5f3b-460d-b9f1-bf0eac48a003 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: bb5907fb-dc42-4963-875c-24ec36ca4fcd kind: Event lastTimestamp: "2026-07-28T01:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:50Z" name: isvc-sklearn-graph-2-predictor.18c6504673ea626d namespace: kserve-ci-e2e-test resourceVersion: "15527" uid: 546d6c98-bf5e-4522-b1c8-ef178b0d7d5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: bb5907fb-dc42-4963-875c-24ec36ca4fcd kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-sklearn-graph-2-predictor.18c6504d71337fb4 namespace: kserve-ci-e2e-test resourceVersion: "16393" uid: 73e5b5ce-28da-4b12-8a12-dd415ae996e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15108" uid: bb5907fb-dc42-4963-875c-24ec36ca4fcd kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-sklearn-graph-2-predictor.18c6504d719bacb9 namespace: kserve-ci-e2e-test resourceVersion: "16394" uid: 201499ea-ea1f-4bdc-b4c4-3e997d4ca225 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14876" uid: 2b24b4c8-e90e-4d3b-aa4a-e24ad4a5cfaa kind: Event lastTimestamp: "2026-07-28T01:37:40Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:40Z" name: isvc-sklearn-graph-2.18c65042f544c420 namespace: kserve-ci-e2e-test resourceVersion: "16152" uid: cacf22dd-bb67-4a17-85bd-d2ec9db8aae4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16148" uid: 2b24b4c8-e90e-4d3b-aa4a-e24ad4a5cfaa kind: Event lastTimestamp: "2026-07-28T01:37:40Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T01:37:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:40Z" name: isvc-sklearn-graph-2.18c6505c7ad23a5d namespace: kserve-ci-e2e-test resourceVersion: "16149" uid: dce8fcec-4553-4c43-a5c2-af31c01536d4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:35:50.665993Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14851" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-b2mj6 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:35:50Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c65042ec918407 namespace: kserve-ci-e2e-test resourceVersion: "14856" uid: 6e647cbd-6e17-4ce0-9e8c-d0cae2f7aaf5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:35:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504306f039bf namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 971ec6e8-1bea-4fcd-8154-ee720e89d376 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:6826da04857ec2521ff8d4edc21f858400a7c6db310668e01c2830086507d99d" already present on machine metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504308449895 namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: e03573bd-8a51-4d82-aeef-0e1b86aaf19a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c650430b70d0ab namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: 34c6ff61-a931-4e2c-b066-501fcfcbf802 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:51Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:35:51Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T01:35:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c650430be8bc8d namespace: kserve-ci-e2e-test resourceVersion: "14937" uid: 12a61405-41c4-40fd-b975-ed7f3255fe72 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:35:57Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T01:35:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504473045361 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: fd5eda9b-ec1f-4bfc-a9a9-929da94b70ff reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 32.869s (32.869s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c1a2aaff9 namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: 113f43a5-9a8a-44a4-81f1-222fb5e38dce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c1ed816be namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: 44c6238f-4c64-4b30-b27c-f410aa6e4fcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c1fa007cd namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: eb164e70-d7ae-4721-b0b6-68bf32ca603b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c200096e0 namespace: kserve-ci-e2e-test resourceVersion: "15338" uid: 7a458121-739f-4d6f-93a6-46bebff94825 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c24c63109 namespace: kserve-ci-e2e-test resourceVersion: "15341" uid: 89fa0a1a-e590-4e82-9a9d-cf99009a4593 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:36:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:30Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c25e3825f namespace: kserve-ci-e2e-test resourceVersion: "15342" uid: 20ab1033-83ed-4245-841b-239915d27151 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T01:36:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:37:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:36:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c6504c2dc61946 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: 20e443e0-d1e6-4e54-ada4-3ade49574626 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c65063850e4cdc namespace: kserve-ci-e2e-test resourceVersion: "16530" uid: 9355cdca-d1fe-4edb-9060-425a2dcc490e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c65063850eb0a5 namespace: kserve-ci-e2e-test resourceVersion: "16534" uid: c992b48b-93d7-42fb-878d-ef4b197bee67 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6 namespace: kserve-ci-e2e-test resourceVersion: "14853" uid: d23e5073-7566-4811-9ffc-96d78ea1d092 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-b2mj6.18c65063b217bf95 namespace: kserve-ci-e2e-test resourceVersion: "16578" uid: 8b0e9ca1-8e9e-4380-af02-08207509aa45 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: a96f298a-b409-4ccf-89c5-4c2dcd21a7ad kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-b2mj6' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c65042ec01fb6f namespace: kserve-ci-e2e-test resourceVersion: "14852" uid: 36d0b2de-bf28-4585-b446-45e88cadd82a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 92792d4a-c004-414d-9a16-439a3b36d719 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: isvc-xgboost-graph-predictor.18c65042e904ae09 namespace: kserve-ci-e2e-test resourceVersion: "14843" uid: 87bb7d7b-64a0-4bd9-82b4-d64e1d9c26c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 2849a356-2c88-45dd-bffb-3b0d25398804 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: isvc-xgboost-graph-predictor.18c650466783f35f namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: f4e0a01f-d8f1-4bf1-90ab-809a627f6d30 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 2849a356-2c88-45dd-bffb-3b0d25398804 kind: Event lastTimestamp: "2026-07-28T01:36:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:35Z" name: isvc-xgboost-graph-predictor.18c6504667ed8e75 namespace: kserve-ci-e2e-test resourceVersion: "15392" uid: 60853131-d96d-4dcb-bc8d-af5c6ad8773f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 2849a356-2c88-45dd-bffb-3b0d25398804 kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-xgboost-graph-predictor.18c65050e3e144f9 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: e99d039f-738e-4752-a555-c084dc33ba90 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:36:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 2849a356-2c88-45dd-bffb-3b0d25398804 kind: Event lastTimestamp: "2026-07-28T01:38:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:05Z" name: isvc-xgboost-graph-predictor.18c65050e44127e9 namespace: kserve-ci-e2e-test resourceVersion: "16389" uid: 427068de-1934-4ec3-8dd0-7b0f5ea211c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14824" uid: 5f585d58-5fae-40d8-8628-114a7962d389 kind: Event lastTimestamp: "2026-07-28T01:37:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:36Z" name: isvc-xgboost-graph.18c65042e93f956d namespace: kserve-ci-e2e-test resourceVersion: "16098" uid: 960c7bb9-0107-49a1-8cc5-d40da74106a8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 5f585d58-5fae-40d8-8628-114a7962d389 kind: Event lastTimestamp: "2026-07-28T01:37:36Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T01:37:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:36Z" name: isvc-xgboost-graph.18c6505b917979e1 namespace: kserve-ci-e2e-test resourceVersion: "16097" uid: 36285c5c-bcff-47bb-a714-27b961f25897 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:38:00.318831Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-b84f4c8c7-vfd7b to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7-vfd7b.18c650611c7ac116 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: 20b8f023-abff-4881-9898-fb25c1ebfcbe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7-vfd7b.18c65061369c438e namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 16f4b519-7100-4cf5-ba62-8376928c782a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7-vfd7b.18c6506137ea1264 namespace: kserve-ci-e2e-test resourceVersion: "16343" uid: e5d1ef98-de88-4986-aff0-9cba0e39cf6b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7-vfd7b.18c650613aa53383 namespace: kserve-ci-e2e-test resourceVersion: "16346" uid: a4cb4f54-b9f3-41ce-8753-79bbbabd32fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7-vfd7b.18c650613b080ea2 namespace: kserve-ci-e2e-test resourceVersion: "16347" uid: 1327929e-c61a-46cd-b7bc-b99ce2fda1b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: model-chainer-b84f4c8c7-vfd7b.18c6506375aebd78 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: a5600e40-acd6-455f-956e-d485541fdb30 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-b84f4c8c7-vfd7b namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 8c0ca180-044d-45d0-b0f1-63ebeba9d2e1 kind: Event lastTimestamp: "2026-07-28T01:38:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:37Z" name: model-chainer-b84f4c8c7-vfd7b.18c65063eb44f776 namespace: kserve-ci-e2e-test resourceVersion: "16837" uid: ee5d1e25-4541-4e1c-b02a-9a3447280f74 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-b84f4c8c7 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: f9a0e3d8-4a1f-41b6-8f09-385109edf818 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: 'Created pod: model-chainer-b84f4c8c7-vfd7b' metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer-b84f4c8c7.18c650611be51eb1 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: 676cd0b3-3788-4dc9-b45e-7fafb9d125b3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 33080ac5-a18a-4a82-9bcf-21902ddda988 kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: Scaled up replica set model-chainer-b84f4c8c7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer.18c650611b028fa1 namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: 6e5f6abe-7b7f-4197-adfd-08f4b5b8d867 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: f00dc47f-e94d-4f05-9846-9231e36f431f kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:00Z" name: model-chainer.18c6506120894c45 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: fc6d7f04-3f1d-4003-b3ae-fc5ad1451c0b reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:35.350863Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21673" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-2a946-76d676585d-c76pf to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:35Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:35Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f4f75953ed namespace: kserve-ci-e2e-test resourceVersion: "21678" uid: f446e1ba-57a9-47a6-bb7d-726ef589346c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:35Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:35Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-2a946-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:35Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f50217f1d3 namespace: kserve-ci-e2e-test resourceVersion: "21688" uid: 852d09ec-0566-42c5-b50f-302d1e9c8246 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:36Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21682" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:36Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:36Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f535561f71 namespace: kserve-ci-e2e-test resourceVersion: "21698" uid: dcc336b9-a5fa-45bd-933a-88517e17a78b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2a946} kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:36Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f536882975 namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: 9fb14758-5def-4945-8ead-38d498067786 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2a946} kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:36Z" message: 'Created container: sequence-graph-2a946' metadata: creationTimestamp: "2026-07-28T01:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:36Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f53991c4e5 namespace: kserve-ci-e2e-test resourceVersion: "21702" uid: 2e19af4a-137d-44ee-abe7-9db4830530cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2a946} kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:36Z" message: Started container sequence-graph-2a946 metadata: creationTimestamp: "2026-07-28T01:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:36Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f53a0e3c6c namespace: kserve-ci-e2e-test resourceVersion: "21703" uid: 4470e499-d9a8-4030-8c7e-38170d35cfed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2a946} kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Stopping container sequence-graph-2a946 metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f751a05ebd namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 58bca983-8f1a-442a-ad33-84c4ad47cb46 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-2a946} kind: Pod name: sequence-graph-2a946-76d676585d-c76pf namespace: kserve-ci-e2e-test resourceVersion: "21675" uid: 24a3f6c4-a95d-4396-b0d2-7e9a93dd37c9 kind: Event lastTimestamp: "2026-07-28T01:49:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:13Z" name: sequence-graph-2a946-76d676585d-c76pf.18c650f802003740 namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: eb00a8cf-95c0-41ef-8c67-2d75f8fffb02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:35Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-2a946-76d676585d namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: efae5d07-4d70-4e50-aed1-f1f20da61044 kind: Event lastTimestamp: "2026-07-28T01:48:35Z" message: 'Created pod: sequence-graph-2a946-76d676585d-c76pf' metadata: creationTimestamp: "2026-07-28T01:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:35Z" name: sequence-graph-2a946-76d676585d.18c650f4f6f29387 namespace: kserve-ci-e2e-test resourceVersion: "21674" uid: 20b624d4-527d-4af9-85ed-ccccecde3b83 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:35Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-2a946 namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 4d775e08-ca58-481f-851c-ea0e9b181a95 kind: Event lastTimestamp: "2026-07-28T01:48:35Z" message: Scaled up replica set sequence-graph-2a946-76d676585d from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:35Z" name: sequence-graph-2a946.18c650f4f63b78d3 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 4c3541c6-59c5-4d32-bf90-ba2e1030d984 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:35Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-2a946 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 09337aed-e7db-4338-a4f6-415797e7c0fd kind: Event lastTimestamp: "2026-07-28T01:48:35Z" message: InferenceGraph [sequence-graph-2a946] is Ready metadata: creationTimestamp: "2026-07-28T01:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:35Z" name: sequence-graph-2a946.18c650f4fb551cee namespace: kserve-ci-e2e-test resourceVersion: "21686" uid: 574e448e-c601-4b97-8848-1711e5559877 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:39:10.615742Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17162" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3d51c-7c5b587fbb-qmdr7 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:39:10Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c650717a808c54 namespace: kserve-ci-e2e-test resourceVersion: "17167" uid: 4fde28c1-f7cb-4a54-806f-52d01c10b9db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:11Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17173" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:39:11Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:39:11Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c6507194610d68 namespace: kserve-ci-e2e-test resourceVersion: "17189" uid: dec473d5-93bf-4d5b-9f48-5da1fc21ee69 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:39:11Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" metadata: creationTimestamp: "2026-07-28T01:39:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:39:11Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c6507195a3906f namespace: kserve-ci-e2e-test resourceVersion: "17191" uid: 5e5cbcba-d31c-4b08-b0e5-48458be64299 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:39:12Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" in 1.817s (1.817s including waiting). Image size: 218232445 bytes.' metadata: creationTimestamp: "2026-07-28T01:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:39:12Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c6507201f467d9 namespace: kserve-ci-e2e-test resourceVersion: "17202" uid: 3f485401-e07d-484a-a119-f9062df9cc41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:39:12Z" message: 'Created container: sequence-graph-3d51c' metadata: creationTimestamp: "2026-07-28T01:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:39:12Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c65072052bb823 namespace: kserve-ci-e2e-test resourceVersion: "17204" uid: 383e7227-eb26-423a-a4ae-7b7285736d90 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:39:12Z" message: Started container sequence-graph-3d51c metadata: creationTimestamp: "2026-07-28T01:39:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:39:12Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c6507205930acf namespace: kserve-ci-e2e-test resourceVersion: "17205" uid: 49eecb9c-d444-4e29-a9e6-85537c63a4b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Stopping container sequence-graph-3d51c metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c650e49ed82073 namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: 5b5ca7ef-332d-440f-8249-f20536f4b880 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3d51c} kind: Pod name: sequence-graph-3d51c-7c5b587fbb-qmdr7 namespace: kserve-ci-e2e-test resourceVersion: "17165" uid: 05a3502d-0259-43ca-8fee-9e12baeb59f7 kind: Event lastTimestamp: "2026-07-28T01:47:54Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:54Z" name: sequence-graph-3d51c-7c5b587fbb-qmdr7.18c650e58cb49b18 namespace: kserve-ci-e2e-test resourceVersion: "21129" uid: 7b73204a-f799-416a-b0de-c81504a4e394 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3d51c-7c5b587fbb namespace: kserve-ci-e2e-test resourceVersion: "17159" uid: 0cb8b411-c6a6-4f5c-98b2-ca9ad52ca089 kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: 'Created pod: sequence-graph-3d51c-7c5b587fbb-qmdr7' metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: sequence-graph-3d51c-7c5b587fbb.18c6507179ddf085 namespace: kserve-ci-e2e-test resourceVersion: "17164" uid: 2a6fe290-8d86-406d-b8e7-10c3219403aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17157" uid: 7439a96e-587a-41a3-a540-6964cc9f80f0 kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: Scaled up replica set sequence-graph-3d51c-7c5b587fbb from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: sequence-graph-3d51c.18c65071792b518a namespace: kserve-ci-e2e-test resourceVersion: "17160" uid: b580293d-3c92-4491-a56d-f1210b8640f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17175" uid: 364fdc69-254d-4a0c-af4a-9c885bab6821 kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: InferenceGraph [sequence-graph-3d51c] is Ready metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:39:10Z" name: sequence-graph-3d51c.18c650717e37593c namespace: kserve-ci-e2e-test resourceVersion: "17177" uid: aae15aa8-0ca9-4f26-a62e-91fcd43b79d5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:39:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: d7c6e802-bb0f-4568-a37c-39b063b29366 kind: Event lastTimestamp: "2026-07-28T01:39:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:55Z" name: sequence-graph-3d51c.18c65074f79e53a0 namespace: kserve-ci-e2e-test resourceVersion: "17515" uid: 97468f8c-9a88-4990-95be-d9a3f40c8da5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:39:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17158" uid: d7c6e802-bb0f-4568-a37c-39b063b29366 kind: Event lastTimestamp: "2026-07-28T01:39:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:39:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:55Z" name: sequence-graph-3d51c.18c65074f8073817 namespace: kserve-ci-e2e-test resourceVersion: "17517" uid: 4233adc9-b477-4cdd-a620-89611a5a7453 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:49:55.685570Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-920f8-67d69d5c5b-bmzql to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:49:55Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c65107aba900a9 namespace: kserve-ci-e2e-test resourceVersion: "22572" uid: ebc21615-650f-4db2-a7c6-c36796e4ea44 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:56Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22576" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:49:56Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:49:56Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c65107c56de824 namespace: kserve-ci-e2e-test resourceVersion: "22590" uid: d7988bb7-035e-40eb-9b83-1ed9e23afafa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-920f8} kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:49:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:56Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c65107c6b2ed87 namespace: kserve-ci-e2e-test resourceVersion: "22593" uid: ce54b531-acd9-4215-8cac-aaea8414b80b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-920f8} kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:49:56Z" message: 'Created container: sequence-graph-920f8' metadata: creationTimestamp: "2026-07-28T01:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:56Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c65107c9d0140c namespace: kserve-ci-e2e-test resourceVersion: "22594" uid: a2fb074e-3863-4df7-9fbe-23163d639d32 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-920f8} kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:49:56Z" message: Started container sequence-graph-920f8 metadata: creationTimestamp: "2026-07-28T01:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:56Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c65107ca47401a namespace: kserve-ci-e2e-test resourceVersion: "22595" uid: dc3f2699-f0c8-4c50-9b70-3786878418e4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-920f8} kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Stopping container sequence-graph-920f8 metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c6517acd37b5d2 namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: eaa5f4fc-9816-4cfd-9050-dff90193faf1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-920f8} kind: Pod name: sequence-graph-920f8-67d69d5c5b-bmzql namespace: kserve-ci-e2e-test resourceVersion: "22569" uid: 5b1f082b-5467-4f38-bf69-ebcce81b199a kind: Event lastTimestamp: "2026-07-28T01:58:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:37Z" name: sequence-graph-920f8-67d69d5c5b-bmzql.18c6517b68bf263d namespace: kserve-ci-e2e-test resourceVersion: "26817" uid: d035c05e-4f5b-4505-84cc-b53514c08ce9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-920f8-67d69d5c5b namespace: kserve-ci-e2e-test resourceVersion: "22563" uid: ccd47e17-4f79-4579-b906-225e62952e11 kind: Event lastTimestamp: "2026-07-28T01:49:55Z" message: 'Created pod: sequence-graph-920f8-67d69d5c5b-bmzql' metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:55Z" name: sequence-graph-920f8-67d69d5c5b.18c65107ab400469 namespace: kserve-ci-e2e-test resourceVersion: "22568" uid: ccd81cad-219f-4f52-b168-4bd3dbf87041 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: 6c992476-5f38-4da8-b55a-bc60cbd7b685 kind: Event lastTimestamp: "2026-07-28T01:49:55Z" message: Scaled up replica set sequence-graph-920f8-67d69d5c5b from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:55Z" name: sequence-graph-920f8.18c65107aa87eeb9 namespace: kserve-ci-e2e-test resourceVersion: "22565" uid: 2c2c8a4d-ca88-4509-889d-e5a52dd304dc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:55Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22579" uid: bdc0b8b1-d070-4792-90b8-a0eab8c02123 kind: Event lastTimestamp: "2026-07-28T01:49:55Z" message: InferenceGraph [sequence-graph-920f8] is Ready metadata: creationTimestamp: "2026-07-28T01:49:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:55Z" name: sequence-graph-920f8.18c65107afa1a4be namespace: kserve-ci-e2e-test resourceVersion: "22581" uid: 5be6264e-347b-4168-a6c7-641784c81757 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 7f28eb52-9284-4c31-9cbd-25887862d3a0 kind: Event lastTimestamp: "2026-07-28T01:50:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:25Z" name: sequence-graph-920f8.18c6510b295b4a37 namespace: kserve-ci-e2e-test resourceVersion: "22816" uid: de047e22-faca-4107-88d1-fbadc116d618 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:50:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 7f28eb52-9284-4c31-9cbd-25887862d3a0 kind: Event lastTimestamp: "2026-07-28T01:50:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:50:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:25Z" name: sequence-graph-920f8.18c6510b29be77fb namespace: kserve-ci-e2e-test resourceVersion: "22817" uid: 518a1443-c94a-43c7-ab1d-8a48ccf525d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:59:51.083012Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27662" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7876a-c666c46dc-zpwsv to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:59:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:59:51Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c651924c1cf2ce namespace: kserve-ci-e2e-test resourceVersion: "27668" uid: 24636755-eb6f-4269-bbf4-85133a441190 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:51Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T01:59:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-7876a-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:51Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c6519252fce2ba namespace: kserve-ci-e2e-test resourceVersion: "27679" uid: a7c69af0-8414-4618-aaeb-1af201e81cbd reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:52Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27671" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T01:59:52Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:59:52Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c6519289d0de62 namespace: kserve-ci-e2e-test resourceVersion: "27688" uid: e3bcdb86-e2eb-4d4f-95d6-eea88d3cbfe7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7876a} kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T01:59:52Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:52Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c651928b0354aa namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: 1c6a8113-2947-4607-ae0b-cd1a94a85404 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7876a} kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T01:59:52Z" message: 'Created container: splitter-graph-7876a' metadata: creationTimestamp: "2026-07-28T01:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:52Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c651928de855a5 namespace: kserve-ci-e2e-test resourceVersion: "27692" uid: e425d861-8c22-41fa-9c14-0e425808faff reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7876a} kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T01:59:52Z" message: Started container splitter-graph-7876a metadata: creationTimestamp: "2026-07-28T01:59:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:52Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c651928e5d74f4 namespace: kserve-ci-e2e-test resourceVersion: "27693" uid: a96e2a29-662d-457b-b9b1-cf327f26bdea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7876a} kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T02:08:05Z" message: Stopping container splitter-graph-7876a metadata: creationTimestamp: "2026-07-28T02:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:05Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c65205771de6e6 namespace: kserve-ci-e2e-test resourceVersion: "31127" uid: 31902224-48ab-480f-8676-70e5dc9b3fd3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T02:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7876a} kind: Pod name: splitter-graph-7876a-c666c46dc-zpwsv namespace: kserve-ci-e2e-test resourceVersion: "27665" uid: 7aa041fb-c1c8-4c82-8cb6-e8478829fb64 kind: Event lastTimestamp: "2026-07-28T02:08:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T02:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:33Z" name: splitter-graph-7876a-c666c46dc-zpwsv.18c652060afb9f67 namespace: kserve-ci-e2e-test resourceVersion: "31361" uid: a2dec52c-4e38-4311-b4f3-59c426cac6e7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7876a-c666c46dc namespace: kserve-ci-e2e-test resourceVersion: "27657" uid: 31f05b17-22dd-4e2b-9fcc-0e116de1676f kind: Event lastTimestamp: "2026-07-28T01:59:51Z" message: 'Created pod: splitter-graph-7876a-c666c46dc-zpwsv' metadata: creationTimestamp: "2026-07-28T01:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:51Z" name: splitter-graph-7876a-c666c46dc.18c651924ba516fe namespace: kserve-ci-e2e-test resourceVersion: "27664" uid: cc14b759-ecbc-462d-8051-87e133320804 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27655" uid: 0d16adc3-c085-4618-8fd4-013968769a9d kind: Event lastTimestamp: "2026-07-28T01:59:51Z" message: Scaled up replica set splitter-graph-7876a-c666c46dc from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:51Z" name: splitter-graph-7876a.18c651924acdb542 namespace: kserve-ci-e2e-test resourceVersion: "27659" uid: 9daeeff9-a1cf-4e03-a886-c97655254858 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27673" uid: 67b944b5-7626-4341-bf28-3f931c13cfdc kind: Event lastTimestamp: "2026-07-28T01:59:51Z" message: InferenceGraph [splitter-graph-7876a] is Ready metadata: creationTimestamp: "2026-07-28T01:59:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:51Z" name: splitter-graph-7876a.18c651924f9aa13f namespace: kserve-ci-e2e-test resourceVersion: "27675" uid: e1d2c8bd-fafe-4433-99f5-da6996bc2284 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 9016f1e9-cb0b-46d2-86bc-f4a6e4fe5db9 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T02:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: splitter-graph-7876a.18c65195c91400a0 namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: 337745ee-a3fc-4192-9391-f0c7853373f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:00:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27656" uid: 9016f1e9-cb0b-46d2-86bc-f4a6e4fe5db9 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T02:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: splitter-graph-7876a.18c65195c97f5021 namespace: kserve-ci-e2e-test resourceVersion: "27803" uid: c4ab47be-4a49-488d-be40-d03a3c08db22 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27800" uid: 9016f1e9-cb0b-46d2-86bc-f4a6e4fe5db9 kind: Event lastTimestamp: "2026-07-28T02:00:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T02:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:21Z" name: splitter-graph-7876a.18c65199480ba6d7 namespace: kserve-ci-e2e-test resourceVersion: "27906" uid: c18ed32c-2feb-4226-b979-93e5b37257d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7876a namespace: kserve-ci-e2e-test resourceVersion: "27800" uid: 9016f1e9-cb0b-46d2-86bc-f4a6e4fe5db9 kind: Event lastTimestamp: "2026-07-28T02:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T02:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:21Z" name: splitter-graph-7876a.18c651994895ce4b namespace: kserve-ci-e2e-test resourceVersion: "27909" uid: d3bda6fa-b6ed-4c56-8a89-23a1cbef1d3a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:58:40.783377Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-bc47b-5b5f84d749-r4bth to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:58:40Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c65181eded9924 namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: fbf402e1-e155-4fa0-823a-ded38f30bd49 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:58:41Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:58:41Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c6518207c8b324 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 09212a87-114d-4cf7-a037-eb4bbbd60fed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bc47b} kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:58:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:41Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c6518209378529 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 1cab2098-7dc5-42b1-b1d2-4c6b54a6c6e3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bc47b} kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:58:41Z" message: 'Created container: splitter-graph-bc47b' metadata: creationTimestamp: "2026-07-28T01:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:41Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c651820c22e4c3 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: d4815535-e513-4986-b9b6-906fd43e9f91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bc47b} kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:58:41Z" message: Started container splitter-graph-bc47b metadata: creationTimestamp: "2026-07-28T01:58:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:41Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c651820caf0fed namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 33b2894c-3abb-40a1-acb6-7dc5048900fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bc47b} kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:58:50Z" message: Stopping container splitter-graph-bc47b metadata: creationTimestamp: "2026-07-28T01:58:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:50Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c65184460578b4 namespace: kserve-ci-e2e-test resourceVersion: "26975" uid: 7a37b456-dc65-4ca7-a3aa-7c766a5f1d68 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-bc47b} kind: Pod name: splitter-graph-bc47b-5b5f84d749-r4bth namespace: kserve-ci-e2e-test resourceVersion: "26864" uid: 77e0c0a6-630f-474c-8d27-bca84a9d28f7 kind: Event lastTimestamp: "2026-07-28T01:59:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:17Z" name: splitter-graph-bc47b-5b5f84d749-r4bth.18c65184c0caa917 namespace: kserve-ci-e2e-test resourceVersion: "27335" uid: 777e22f5-ec31-45f8-a73f-cf4af74bd93b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-bc47b-5b5f84d749 namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: c2a05581-6b26-4c9f-8b4a-833453b5e6b5 kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'Created pod: splitter-graph-bc47b-5b5f84d749-r4bth' metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: splitter-graph-bc47b-5b5f84d749.18c65181ed82fac1 namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: 938799fd-9f05-498e-873b-e7a97943a214 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: daf4e5ea-f129-47b6-a24e-78d2947a82af kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: Scaled up replica set splitter-graph-bc47b-5b5f84d749 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: splitter-graph-bc47b.18c65181ec94562d namespace: kserve-ci-e2e-test resourceVersion: "26858" uid: 6bc85f98-c3da-4f20-a72c-178ffed4fae1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 16e4af24-0431-4ec9-aac0-dd59b457babc kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: InferenceGraph [splitter-graph-bc47b] is Ready metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:40Z" name: splitter-graph-bc47b.18c65181f120d00d namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 6dec2ca4-7c96-414f-b270-689f469c588f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:46:55.601231Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-02c81-predictor-5b98d76d87-zqtqz to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650ddbdcc03ee namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: b8c30244-0d6f-462a-ab28-ee419ce0e299 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650ddd7d96990 namespace: kserve-ci-e2e-test resourceVersion: "20478" uid: fb0bba12-2f54-453b-8d92-f59e7446444b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650ddd92c5e97 namespace: kserve-ci-e2e-test resourceVersion: "20481" uid: 5214fac1-ab02-46b3-b299-7570ec497466 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650dddc14aeea namespace: kserve-ci-e2e-test resourceVersion: "20484" uid: 86cca34b-76db-4793-9ded-4935b98edc74 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650dddc943505 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: fa4ee587-6acd-4d0c-ae13-bc37e7f16e8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650dddca0652e namespace: kserve-ci-e2e-test resourceVersion: "20487" uid: 825ae9ea-8d88-4973-9a19-e76416f616fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650dddf39844b namespace: kserve-ci-e2e-test resourceVersion: "20488" uid: e57e1f9c-cbf3-4f75-9e9d-bb611416259e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:46:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:56Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650dddfb2d3d8 namespace: kserve-ci-e2e-test resourceVersion: "20490" uid: 3fee6184-aaf1-4622-91b6-af61560d1ef0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:46:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:47:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:46:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:42Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650de0fc23852 namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: 34ded206-034f-46ab-8f01-b292da7c758c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:15Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650f06b80342e namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 9dc7801d-290e-4c91-94a2-77c78097e418 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:15Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650f06b80a096 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: ecb08295-5319-492e-a84e-bf217e8054b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz namespace: kserve-ci-e2e-test resourceVersion: "20417" uid: e1c45a3e-cc55-4400-99ea-f97663c7f77c kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz.18c650f0ebef89d5 namespace: kserve-ci-e2e-test resourceVersion: "21497" uid: ead63398-7969-4e22-919c-a44e29b7d389 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-02c81-predictor-5b98d76d87 namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: 79077150-4c33-4389-8426-97eae3cbb7ff kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'Created pod: success-200-isvc-02c81-predictor-5b98d76d87-zqtqz' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-02c81-predictor-5b98d76d87.18c650ddbd0ea03c namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: 346d3ad3-ca87-4d8c-9f17-1533b58dbfcc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: d477a97c-de65-47c1-88b8-87dc1216c0ce kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Scaled up replica set success-200-isvc-02c81-predictor-5b98d76d87 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-02c81-predictor.18c650ddb949f721 namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: 33a946c2-b35d-4ad3-a2ae-ab987d498b51 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: c35d818d-b79a-4d51-9536-782a0f903984 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-02c81-predictor.18c650e138177efd namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 1e85aa58-4c8a-46cd-9995-c5ce992ce779 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20395" uid: c35d818d-b79a-4d51-9536-782a0f903984 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-02c81-predictor.18c650e13879bd38 namespace: kserve-ci-e2e-test resourceVersion: "20840" uid: 997a0bd5-5a32-4f24-b7e6-1f846b9769bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: c35d818d-b79a-4d51-9536-782a0f903984 kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: success-200-isvc-02c81-predictor.18c650e835bd1553 namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: 11478070-dcfe-4747-8052-dc741adf7e7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-02c81-predictor namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: c35d818d-b79a-4d51-9536-782a0f903984 kind: Event lastTimestamp: "2026-07-28T01:48:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:10Z" name: success-200-isvc-02c81-predictor.18c650e8362324a7 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: 14662960-cc37-4665-9570-5c080ad5ddf9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 2ddba3cc-439e-4872-88c2-e289f00a000c kind: Event lastTimestamp: "2026-07-28T01:47:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:53Z" name: success-200-isvc-02c81.18c650ddb963813f namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 30033872-aa6d-462a-8d1a-7f269d2ae41b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 2ddba3cc-439e-4872-88c2-e289f00a000c kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-02c81": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02c81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-02c81.18c650ddb9dcb9b7 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: ed88286e-0be1-4a65-b015-591f15677902 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 2ddba3cc-439e-4872-88c2-e289f00a000c kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-02c81": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-02c81.18c650ddb9dd839f namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: 773bc26b-5326-4e0c-b398-f0e60133a37a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-02c81 namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 2ddba3cc-439e-4872-88c2-e289f00a000c kind: Event lastTimestamp: "2026-07-28T01:47:53Z" message: InferenceService [success-200-isvc-02c81] is Ready metadata: creationTimestamp: "2026-07-28T01:47:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:53Z" name: success-200-isvc-02c81.18c650eb1fefe490 namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: df65a91b-7765-4f74-a1dc-891e742bcf3e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:47:25.349941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-2a946-predictor-86c64754c9-psq5l to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4aaf55256 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: 07c98afd-970e-420e-97fd-358a075fd485 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4c4df5a60 namespace: kserve-ci-e2e-test resourceVersion: "20850" uid: 8103e427-e91a-482d-bcf6-450d843c8798 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4c62fbf70 namespace: kserve-ci-e2e-test resourceVersion: "20852" uid: fbd407be-5d27-4475-b9b3-217cbae3ec66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4c9436337 namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 059e4326-e3e8-4c62-81fd-67ee16a3a4a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4c9b99c5a namespace: kserve-ci-e2e-test resourceVersion: "20854" uid: 4a3185ad-6804-4cf5-ac27-c60014ff54be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4c9c4f541 namespace: kserve-ci-e2e-test resourceVersion: "20856" uid: 58831ff3-a452-4634-b141-16a8716a23cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4ce67de31 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 8b11cc0e-53b6-47a5-9e76-ecbb185e94b7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4cf01bcca namespace: kserve-ci-e2e-test resourceVersion: "20859" uid: cd1081c3-6b99-4c54-9255-06390ced3880 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:47:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:48:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:47:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:12Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650e4d75e42e1 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: 8efa2434-c6dc-486c-9c08-99e7e639b2c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650f7588a7be7 namespace: kserve-ci-e2e-test resourceVersion: "21817" uid: 3952d7c5-871f-4c1b-b638-255cef983d72 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650f7588b0912 namespace: kserve-ci-e2e-test resourceVersion: "21821" uid: 7753b249-4b52-4cc5-8ef5-c7de53d0eadb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-2a946-predictor-86c64754c9-psq5l namespace: kserve-ci-e2e-test resourceVersion: "20799" uid: 5b8eaec4-835a-43cf-b192-aa4bbe5ef0e7 kind: Event lastTimestamp: "2026-07-28T01:48:47Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:47Z" name: success-200-isvc-2a946-predictor-86c64754c9-psq5l.18c650f7b3875179 namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: 3d3631ef-7b81-4a74-aa9f-efb03b6398f0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-2a946-predictor-86c64754c9 namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 78af4a64-c4b0-4447-85e6-5e5002011802 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Created pod: success-200-isvc-2a946-predictor-86c64754c9-psq5l' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor-86c64754c9.18c650e4aa89c77e namespace: kserve-ci-e2e-test resourceVersion: "20798" uid: 72fac0d3-a75f-4318-a16d-85ba85179af0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20774" uid: 04ca751f-36c0-46c4-8193-10290ff5509d kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Scaled up replica set success-200-isvc-2a946-predictor-86c64754c9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946-predictor.18c650e4a8094363 namespace: kserve-ci-e2e-test resourceVersion: "20785" uid: 5d168d28-d8c3-4574-b668-6d5d936b890e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 77fe4111-e15d-4dc3-b97d-c2669de56f2c kind: Event lastTimestamp: "2026-07-28T01:47:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:55Z" name: success-200-isvc-2a946-predictor.18c650e82722384c namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: c8b0cc77-5b41-4af6-9db2-e857dc04059d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:47:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 77fe4111-e15d-4dc3-b97d-c2669de56f2c kind: Event lastTimestamp: "2026-07-28T01:47:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:47:55Z" name: success-200-isvc-2a946-predictor.18c650e82791d6e0 namespace: kserve-ci-e2e-test resourceVersion: "21139" uid: 6a0280a2-3843-4ef5-8ead-44cfcc28775f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:48:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 77fe4111-e15d-4dc3-b97d-c2669de56f2c kind: Event lastTimestamp: "2026-07-28T01:48:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:40Z" name: success-200-isvc-2a946-predictor.18c650ef25719536 namespace: kserve-ci-e2e-test resourceVersion: "21734" uid: 29f83bc5-aa19-4905-9f43-f29f46892438 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:48:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-2a946-predictor namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 77fe4111-e15d-4dc3-b97d-c2669de56f2c kind: Event lastTimestamp: "2026-07-28T01:48:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:40Z" name: success-200-isvc-2a946-predictor.18c650ef25d27ba5 namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 9de9145d-97ef-4a71-a6c2-4f113d766e3d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 021e7631-3979-4586-a45c-1912d6ac5627 kind: Event lastTimestamp: "2026-07-28T01:48:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:22Z" name: success-200-isvc-2a946.18c650e4a820929a namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 45b1f1cc-c0a2-40de-83ad-490590bc0f09 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 021e7631-3979-4586-a45c-1912d6ac5627 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'Failed to update status for InferenceService "success-200-isvc-2a946": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2a946": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946.18c650e4a8a061bc namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: 7337dcfb-a508-410c-9de1-5f915953b8e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 021e7631-3979-4586-a45c-1912d6ac5627 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-2a946": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-2a946.18c650e4a8a0ad0e namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 1c8d3f60-1dec-4564-96d5-1f31bcb3b837 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-2a946 namespace: kserve-ci-e2e-test resourceVersion: "21551" uid: 021e7631-3979-4586-a45c-1912d6ac5627 kind: Event lastTimestamp: "2026-07-28T01:48:22Z" message: InferenceService [success-200-isvc-2a946] is Ready metadata: creationTimestamp: "2026-07-28T01:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:48:22Z" name: success-200-isvc-2a946.18c650f1e666072f namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: f4e12cf7-5509-4156-a898-9787c2e62b7f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:37:30.756744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a3a71295b namespace: kserve-ci-e2e-test resourceVersion: "15953" uid: 58b4b6db-36ac-405e-97f7-3c1397403c02 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-33843-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a44c1fda7 namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: b5b9e040-a9c2-457e-a894-482ab2464dfa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15964" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a7864c116 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 81713f78-d5ca-4c9a-afca-c78d8d39bcde reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a79cf6f4f namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: f5a7acf9-1647-4fb9-b20d-e9e39f57125f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a7d1d3efe namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: 8ae5f590-33d4-4f46-8ca4-900d2a5417ee reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a7d8c0528 namespace: kserve-ci-e2e-test resourceVersion: "16026" uid: 1e6fd64c-e4ca-4796-a36e-c995d81ac21c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a7d9714cf namespace: kserve-ci-e2e-test resourceVersion: "16027" uid: c33e6464-63dd-49e5-8f4b-b21220c925df reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a80678668 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: 4fba7bd3-a2da-41c2-aefd-ab752dae3f97 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:37:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:37:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:31Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505a80e8580a namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 1ad797f2-b188-45a9-9f30-fdff2dccdbc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T01:37:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:46:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:37:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:58Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c6505aad5e77eb namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: 6f9e8a92-930f-45ae-a2f6-52ea36397c3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c650ddba063241 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 303ebb7a-845c-4c5f-9cda-88c2569d65a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:55Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c650ddba06bdc5 namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: 7055d85b-c6db-473e-8a3c-94f34d77ebd7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: ffacdfd9-fb25-4a70-b282-36051a7dcafc kind: Event lastTimestamp: "2026-07-28T01:46:58Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:58Z" name: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f.18c650de75c88569 namespace: kserve-ci-e2e-test resourceVersion: "20524" uid: 5cf7cb34-1dce-4d4c-bd02-b62624b51c82 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-33843-predictor-78f5cf5cd9 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 4574f7ad-6929-471b-915d-bb752682d037 kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'Created pod: success-200-isvc-33843-predictor-78f5cf5cd9-vzq7f' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843-predictor-78f5cf5cd9.18c6505a3a031034 namespace: kserve-ci-e2e-test resourceVersion: "15949" uid: 50da319d-6f4b-4aa5-81d1-2120ab084380 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: df4d8273-b878-4a12-836b-d90fac2b477f kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Scaled up replica set success-200-isvc-33843-predictor-78f5cf5cd9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843-predictor.18c6505a36994d57 namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: 523c01f5-d582-49ce-809c-cdd3e71a6702 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 760f81dc-bb3a-4e34-8bba-12e47cfea4af kind: Event lastTimestamp: "2026-07-28T01:37:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:45Z" name: success-200-isvc-33843-predictor.18c6505db512374f namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 6997ab05-4406-4a1d-9c0a-9558024819ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "15932" uid: 760f81dc-bb3a-4e34-8bba-12e47cfea4af kind: Event lastTimestamp: "2026-07-28T01:37:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:45Z" name: success-200-isvc-33843-predictor.18c6505db5797679 namespace: kserve-ci-e2e-test resourceVersion: "16187" uid: a69f572a-9ed6-4221-a39f-5bbca8db0a9e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 760f81dc-bb3a-4e34-8bba-12e47cfea4af kind: Event lastTimestamp: "2026-07-28T01:39:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:15Z" name: success-200-isvc-33843-predictor.18c6506134295126 namespace: kserve-ci-e2e-test resourceVersion: "17226" uid: 6112b084-2af0-436f-8e90-7bc41de9aaa5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-33843-predictor namespace: kserve-ci-e2e-test resourceVersion: "16186" uid: 760f81dc-bb3a-4e34-8bba-12e47cfea4af kind: Event lastTimestamp: "2026-07-28T01:39:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:15Z" name: success-200-isvc-33843-predictor.18c65061348fbd68 namespace: kserve-ci-e2e-test resourceVersion: "17227" uid: 8dad7e81-aafe-4614-a7ac-09de2703296d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 796bb626-786e-48f4-a33b-08b86b37e84a kind: Event lastTimestamp: "2026-07-28T01:38:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:28Z" name: success-200-isvc-33843.18c6505a36b6c408 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 5a6eac77-20fa-4845-8a7d-cc56dc1f822f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 796bb626-786e-48f4-a33b-08b86b37e84a kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'Failed to update status for InferenceService "success-200-isvc-33843": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-33843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843.18c6505a37610595 namespace: kserve-ci-e2e-test resourceVersion: "15942" uid: 9fe91b4e-817f-47be-863a-ceea2f87dfe7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 796bb626-786e-48f4-a33b-08b86b37e84a kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-33843": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-33843.18c6505a37614d71 namespace: kserve-ci-e2e-test resourceVersion: "15945" uid: a0682d57-0c49-4e60-afc3-1bbaf2279fff reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-33843 namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: 796bb626-786e-48f4-a33b-08b86b37e84a kind: Event lastTimestamp: "2026-07-28T01:38:28Z" message: InferenceService [success-200-isvc-33843] is Ready metadata: creationTimestamp: "2026-07-28T01:38:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:28Z" name: success-200-isvc-33843.18c65067bcbf1e9c namespace: kserve-ci-e2e-test resourceVersion: "16765" uid: 78083af6-0b0e-4bdb-90a6-b1b4361d7f8c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:38:10.597236Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16500" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:38:10Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063811ecc87 namespace: kserve-ci-e2e-test resourceVersion: "16507" uid: 4f1b73b6-8711-481d-8239-9272bae687fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-3d51c-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:10Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c650638bb47338 namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 5ed289e6-34c0-4d62-a281-203e7448d56d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16524" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063beeb2632 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 20d00f9e-ba9c-4d6d-b3f3-71b1f792272e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c0317386 namespace: kserve-ci-e2e-test resourceVersion: "16585" uid: d8bd8a7f-1492-4882-8af3-f2123b4f30fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c4648cf1 namespace: kserve-ci-e2e-test resourceVersion: "16588" uid: 674b59ae-e176-45dd-86ac-c9e93d83ed89 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c564c325 namespace: kserve-ci-e2e-test resourceVersion: "16589" uid: 467c8e29-01d8-40ee-8c93-79f92c43215e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c571255a namespace: kserve-ci-e2e-test resourceVersion: "16591" uid: c3259cc5-9d79-455c-bf81-4c97ed764ffa reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c863c43f namespace: kserve-ci-e2e-test resourceVersion: "16592" uid: db57b5bb-3572-47ee-b3d6-840f33692c9b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:11Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c65063c8ce08e3 namespace: kserve-ci-e2e-test resourceVersion: "16593" uid: 156867b0-2a2d-4218-a150-4f73b4d460a3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:38:48Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:48Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c6506408605018 namespace: kserve-ci-e2e-test resourceVersion: "16973" uid: 91273cc1-f1e9-40c6-b0a0-13bba60d712a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c650e4a52a83d8 namespace: kserve-ci-e2e-test resourceVersion: "20761" uid: 1169a7a4-87e0-4ebc-b4a0-cc9e086b3dd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:47:25Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:47:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:25Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c650e4a52b4add namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: 1ad9cecd-cdb8-4132-9666-3eb403f0bc2d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:47:29Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-28T01:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:29Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c650e5b88193c7 namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: 1ddf18bb-608c-4052-be09-976e4bb87ec8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:47:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np namespace: kserve-ci-e2e-test resourceVersion: "16505" uid: 9e66766a-1d6b-4d63-9ea2-6bfd489b8086 kind: Event lastTimestamp: "2026-07-28T01:47:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: i/o timeout' metadata: creationTimestamp: "2026-07-28T01:47:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:29Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np.18c650e5b8eaec71 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: 40872b23-ffba-4e03-8df5-454e0339fc65 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3d51c-predictor-7d7fc68cc5 namespace: kserve-ci-e2e-test resourceVersion: "16488" uid: de725cb0-4f6e-455e-94a9-2bc6418f9b47 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: 'Created pod: success-200-isvc-3d51c-predictor-7d7fc68cc5-fz2np' metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:10Z" name: success-200-isvc-3d51c-predictor-7d7fc68cc5.18c6506380550153 namespace: kserve-ci-e2e-test resourceVersion: "16504" uid: 5524045a-a1a8-4b28-97ed-a17782775eee reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: 1cd524ab-60e4-46fe-9e0e-c59e8ba88f14 kind: Event lastTimestamp: "2026-07-28T01:38:10Z" message: Scaled up replica set success-200-isvc-3d51c-predictor-7d7fc68cc5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:10Z" name: success-200-isvc-3d51c-predictor.18c650637e6a4b03 namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: ba60b98f-9c30-4bdf-94a3-3a885ecc0f8d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: a525000b-53d1-4a29-9cdf-c270cd623285 kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: success-200-isvc-3d51c-predictor.18c65066fcc0b4ce namespace: kserve-ci-e2e-test resourceVersion: "16865" uid: 4ecca91c-c781-4104-a9e3-1dcf330ee49c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:38:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16486" uid: a525000b-53d1-4a29-9cdf-c270cd623285 kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: success-200-isvc-3d51c-predictor.18c65066fd25926b namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: a1019ed3-60b4-422b-9cad-a707c355d682 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: a525000b-53d1-4a29-9cdf-c270cd623285 kind: Event lastTimestamp: "2026-07-28T01:39:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:40Z" name: success-200-isvc-3d51c-predictor.18c6506dfaf06306 namespace: kserve-ci-e2e-test resourceVersion: "17416" uid: 7bfbf453-9257-4ffd-9b1b-554465de8cca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3d51c-predictor namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: a525000b-53d1-4a29-9cdf-c270cd623285 kind: Event lastTimestamp: "2026-07-28T01:39:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:40Z" name: success-200-isvc-3d51c-predictor.18c6506dfb562bc3 namespace: kserve-ci-e2e-test resourceVersion: "17417" uid: 53ce5e3e-73ca-4f40-9d8b-ded6005f2ee0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:38:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: 5d9cf7ae-a0f9-4183-8308-cd9f50d87e11 kind: Event lastTimestamp: "2026-07-28T01:38:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:58Z" name: success-200-isvc-3d51c.18c650637e844700 namespace: kserve-ci-e2e-test resourceVersion: "17071" uid: 31b5f2f0-f16f-4742-a5ab-5159358fef3f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3d51c namespace: kserve-ci-e2e-test resourceVersion: "17069" uid: 5d9cf7ae-a0f9-4183-8308-cd9f50d87e11 kind: Event lastTimestamp: "2026-07-28T01:38:58Z" message: InferenceService [success-200-isvc-3d51c] is Ready metadata: creationTimestamp: "2026-07-28T01:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:58Z" name: success-200-isvc-3d51c.18c6506ec3a70f26 namespace: kserve-ci-e2e-test resourceVersion: "17070" uid: af496e2c-0207-43f8-94a5-db22c2e8ac24 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:15.889649Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21411" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:15Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f06f5c1029 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: d9577d6c-c2f3-4fa8-bc70-1285a0e797c8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-5a4a0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:16Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f07a6a5bc0 namespace: kserve-ci-e2e-test resourceVersion: "21457" uid: 08991fa1-a97a-4dbe-9121-10a74c519f35 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:16Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:16Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0ad13d7f0 namespace: kserve-ci-e2e-test resourceVersion: "21471" uid: d91769fe-5cec-4a6c-9af3-1d8da4e09fa8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:16Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:16Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0ae6d18dd namespace: kserve-ci-e2e-test resourceVersion: "21473" uid: 5196a8c4-c917-44a5-906b-43e56a1c8773 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0b1fbc4e3 namespace: kserve-ci-e2e-test resourceVersion: "21474" uid: 75f74363-c23d-47e0-bf2c-69badcf01468 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0b274ae77 namespace: kserve-ci-e2e-test resourceVersion: "21476" uid: 2f952bc9-b8db-44f7-aef3-03e7c85d447a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0b27f9e22 namespace: kserve-ci-e2e-test resourceVersion: "21477" uid: 931afd7b-4d4e-4655-88a1-656fd290cee6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0b5ccb26a namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 448a0f74-2602-4bd9-b2d6-ee1ae218858e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:48:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:17Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f0b64444bc namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 68e19222-3a5a-4936-aec6-53ebb6ac4aa4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:48:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:49:04Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:49:04Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c650f1006532d8 namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: 9b288977-1f62-4165-b428-14d24cdae146 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c65173f2fec0bf namespace: kserve-ci-e2e-test resourceVersion: "26102" uid: 41e0f5d9-145c-4777-9987-a0dc42f2519e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh namespace: kserve-ci-e2e-test resourceVersion: "21414" uid: a1cacbd6-d19e-4f79-99af-ea1ce733efc7 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh.18c65173f2ff3e2a namespace: kserve-ci-e2e-test resourceVersion: "26103" uid: 4aa005bf-723a-40d7-b1f6-e5e70152203a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-5a4a0-predictor-777c9d8757 namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: 4e4d6227-3aa6-486c-9941-5ffb340b3a67 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: 'Created pod: success-200-isvc-5a4a0-predictor-777c9d8757-wdkqh' metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:15Z" name: success-200-isvc-5a4a0-predictor-777c9d8757.18c650f06ecdfd3f namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 52f235ac-eeba-4b4b-9aba-75e7d52293e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: ecaf319e-bfdb-41d0-97d6-ef55d898f2b4 kind: Event lastTimestamp: "2026-07-28T01:48:15Z" message: Scaled up replica set success-200-isvc-5a4a0-predictor-777c9d8757 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:15Z" name: success-200-isvc-5a4a0-predictor.18c650f06cd316b9 namespace: kserve-ci-e2e-test resourceVersion: "21400" uid: b1aa4122-f3c6-4229-91f1-ca2e45ecc67b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 82e38aa6-3426-45d7-8182-016afe0a84c1 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-5a4a0-predictor.18c650f3eb53b7f3 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: abc12770-7d78-49ff-87c7-21926417e031 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:48:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21393" uid: 82e38aa6-3426-45d7-8182-016afe0a84c1 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-5a4a0-predictor.18c650f3ebb99e60 namespace: kserve-ci-e2e-test resourceVersion: "21890" uid: a6f9c1b6-e0ad-44a7-a74a-28c55e4306dc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 82e38aa6-3426-45d7-8182-016afe0a84c1 kind: Event lastTimestamp: "2026-07-28T01:49:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:30Z" name: success-200-isvc-5a4a0-predictor.18c650fae9116d2a namespace: kserve-ci-e2e-test resourceVersion: "22345" uid: a7ad770d-f1dd-47f9-af27-5bfe2c7a0c46 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-5a4a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: 82e38aa6-3426-45d7-8182-016afe0a84c1 kind: Event lastTimestamp: "2026-07-28T01:49:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:30Z" name: success-200-isvc-5a4a0-predictor.18c650fae97d30b9 namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: e4d15e20-bf53-4534-b3b0-2d1c7e180eda reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 699b74fe-d0da-4518-8c27-2028de8ce623 kind: Event lastTimestamp: "2026-07-28T01:49:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:48:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:14Z" name: success-200-isvc-5a4a0.18c650f06ce6f518 namespace: kserve-ci-e2e-test resourceVersion: "22175" uid: 8bc24ba1-f6e8-441b-95a0-999408531424 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-5a4a0 namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: 699b74fe-d0da-4518-8c27-2028de8ce623 kind: Event lastTimestamp: "2026-07-28T01:49:14Z" message: InferenceService [success-200-isvc-5a4a0] is Ready metadata: creationTimestamp: "2026-07-28T01:49:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:14Z" name: success-200-isvc-5a4a0.18c650fe0f6f8f68 namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: fd1f2e94-c4b9-4500-8f2f-a2cce15d1a72 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:58:10.348408Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26485" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-704b9-predictor-5774cf759-k29gm to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517ad7dce519 namespace: kserve-ci-e2e-test resourceVersion: "26492" uid: f83ca07c-6e86-44de-b2ec-48d99392058e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-704b9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517ae5fe69bb namespace: kserve-ci-e2e-test resourceVersion: "26533" uid: b8d3547b-52fb-4711-a476-ee66f519539c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26500" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b15aaffb8 namespace: kserve-ci-e2e-test resourceVersion: "26562" uid: 2243c003-c6d7-46b9-b52a-946b14a12027 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b17040327 namespace: kserve-ci-e2e-test resourceVersion: "26564" uid: 354b212e-dfff-4dc1-b38a-6aa86982c9ec reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b1a4011ba namespace: kserve-ci-e2e-test resourceVersion: "26565" uid: 2376d011-eb70-4784-b9db-84ec6b8b70ed reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b1ab9a2c4 namespace: kserve-ci-e2e-test resourceVersion: "26566" uid: e923014c-7c40-42d2-be3a-460d76bb78b7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b1ac45527 namespace: kserve-ci-e2e-test resourceVersion: "26567" uid: 15ce9300-27df-4996-ab44-7e9c31522477 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b1ed31d7c namespace: kserve-ci-e2e-test resourceVersion: "26568" uid: 2714fe03-7a59-49fa-a227-adfadb723a8c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:11Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b1f50d803 namespace: kserve-ci-e2e-test resourceVersion: "26569" uid: 04d3332c-62e7-4f55-856f-e29166337032 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T01:58:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:57Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6517b2cd82fc1 namespace: kserve-ci-e2e-test resourceVersion: "27164" uid: 898ca131-0873-4fa2-af58-bc09456ffc12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T02:15:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T02:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:39Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6526f285f40fb namespace: kserve-ci-e2e-test resourceVersion: "34290" uid: bf09c6ea-0a5b-4bd4-bf1d-f33f3b1bf3b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T02:15:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T02:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:39Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6526f285fdad4 namespace: kserve-ci-e2e-test resourceVersion: "34291" uid: 7efe6afb-909c-4d6d-9b9b-872e2b7a3bf1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-704b9-predictor-5774cf759-k29gm namespace: kserve-ci-e2e-test resourceVersion: "26488" uid: 8b088f90-c19d-4e15-acf4-210df4ecf4b4 kind: Event lastTimestamp: "2026-07-28T02:15:42Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T02:15:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:42Z" name: success-200-isvc-704b9-predictor-5774cf759-k29gm.18c6526fe19cc9c2 namespace: kserve-ci-e2e-test resourceVersion: "34359" uid: ca4df666-84b5-4058-9c87-be7ed8695bc3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-704b9-predictor-5774cf759 namespace: kserve-ci-e2e-test resourceVersion: "26469" uid: 2784958d-369a-4396-ad26-b0ae1093cebf kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'Created pod: success-200-isvc-704b9-predictor-5774cf759-k29gm' metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-704b9-predictor-5774cf759.18c6517ad767a8a2 namespace: kserve-ci-e2e-test resourceVersion: "26487" uid: e91d5ca8-d89a-48cf-b11c-1b1d9de4fe8c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26465" uid: 3302be91-2113-4837-bb89-00a4c3c0d43e kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Scaled up replica set success-200-isvc-704b9-predictor-5774cf759 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-704b9-predictor.18c6517ad57bb6c3 namespace: kserve-ci-e2e-test resourceVersion: "26475" uid: 4330f73b-fa0e-4b41-8858-eccc81aee5cb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 2becbfc6-13a1-42ec-ba6d-3c271c336d34 kind: Event lastTimestamp: "2026-07-28T01:58:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:25Z" name: success-200-isvc-704b9-predictor.18c6517e53ebed5f namespace: kserve-ci-e2e-test resourceVersion: "26697" uid: 80e7eb80-d8a0-4bf1-a5fb-86d73331167c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26468" uid: 2becbfc6-13a1-42ec-ba6d-3c271c336d34 kind: Event lastTimestamp: "2026-07-28T01:58:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:25Z" name: success-200-isvc-704b9-predictor.18c6517e544f9982 namespace: kserve-ci-e2e-test resourceVersion: "26699" uid: 74d50179-c998-4797-a52f-71f9704d835f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 2becbfc6-13a1-42ec-ba6d-3c271c336d34 kind: Event lastTimestamp: "2026-07-28T01:59:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:25Z" name: success-200-isvc-704b9-predictor.18c65181d350dfc4 namespace: kserve-ci-e2e-test resourceVersion: "27437" uid: 46e502eb-3e3a-41e6-b80f-d1c4a83e14ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T01:58:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-704b9-predictor namespace: kserve-ci-e2e-test resourceVersion: "26698" uid: 2becbfc6-13a1-42ec-ba6d-3c271c336d34 kind: Event lastTimestamp: "2026-07-28T01:59:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:25Z" name: success-200-isvc-704b9-predictor.18c65181d3b2e316 namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: dab1be55-52a0-464f-b11d-ea5f2a0deba2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: cf7f7ccb-3b9a-457f-a3c8-fd9f1aa0a95e kind: Event lastTimestamp: "2026-07-28T01:59:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:07Z" name: success-200-isvc-704b9.18c6517ad5809537 namespace: kserve-ci-e2e-test resourceVersion: "27262" uid: 13b2f596-6c4b-4e12-bafe-dd87e73397d9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: cf7f7ccb-3b9a-457f-a3c8-fd9f1aa0a95e kind: Event lastTimestamp: "2026-07-28T01:59:07Z" message: InferenceService [success-200-isvc-704b9] is Ready metadata: creationTimestamp: "2026-07-28T01:59:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:07Z" name: success-200-isvc-704b9.18c651883c906185 namespace: kserve-ci-e2e-test resourceVersion: "27259" uid: 740768c4-41a8-457f-b424-87be6d879e0c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:58:51.046136Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27028" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7876a-predictor-7b564495c-rrcc7 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c6518451a2dd5e namespace: kserve-ci-e2e-test resourceVersion: "27034" uid: 5961d75f-d26e-4657-bb9c-d7b51e5f31b0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7876a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c651845acc74e3 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 15ab670a-c9d2-4b31-8b81-323402a325b6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27036" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c651848f95b707 namespace: kserve-ci-e2e-test resourceVersion: "27098" uid: e7ad5231-2776-4317-bf36-9c644d6fe595 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c6518490ef65ad namespace: kserve-ci-e2e-test resourceVersion: "27100" uid: f2ee5d92-b30a-4f08-9845-0b9bc5ddcc49 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c6518494c84493 namespace: kserve-ci-e2e-test resourceVersion: "27101" uid: 30e13055-c1ee-46df-ba05-b317de5cef09 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c651849547d227 namespace: kserve-ci-e2e-test resourceVersion: "27102" uid: 3084d408-6b1b-41c8-be51-0f0246b4390a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c651849554a6e8 namespace: kserve-ci-e2e-test resourceVersion: "27103" uid: c3c8641f-4c01-41e5-ac48-c2b6308669cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c651849954584f namespace: kserve-ci-e2e-test resourceVersion: "27104" uid: 1954260e-b332-4839-a6b3-e41cbcef0b14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T01:58:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:52Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c6518499de4399 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: 01d44ef8-bda2-4e51-aa16-aec3dc5ba272 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:58:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T02:08:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:08Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c65184c3952ce5 namespace: kserve-ci-e2e-test resourceVersion: "31187" uid: 73954e27-3536-4c6c-9483-54b9f6e5fa99 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T02:08:05Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T02:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:05Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c652057f530c6f namespace: kserve-ci-e2e-test resourceVersion: "31143" uid: 6ea91bc9-7adf-4536-bb77-a1f4a09043e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T02:08:05Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T02:08:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:05Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c652057f53871a namespace: kserve-ci-e2e-test resourceVersion: "31144" uid: 18996d6d-ae10-4e7d-9f7a-fcfa1170bd59 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:08:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7876a-predictor-7b564495c-rrcc7 namespace: kserve-ci-e2e-test resourceVersion: "27031" uid: 9fef3c61-9e7b-4a3c-9c37-f15a06fcf3ad kind: Event lastTimestamp: "2026-07-28T02:08:08Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T02:08:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:08:08Z" name: success-200-isvc-7876a-predictor-7b564495c-rrcc7.18c652063801f052 namespace: kserve-ci-e2e-test resourceVersion: "31186" uid: 1eb243a5-ba16-4a89-a72e-05abc6765577 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7876a-predictor-7b564495c namespace: kserve-ci-e2e-test resourceVersion: "27013" uid: fe0eb8a6-127c-455c-a7e0-2515a9ed9a5e kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Created pod: success-200-isvc-7876a-predictor-7b564495c-rrcc7' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a-predictor-7b564495c.18c6518451060b6e namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 99300227-76ea-4a63-97f4-45c3b1160668 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27004" uid: b98528ca-5d57-4cc9-b09e-abad966c23f6 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Scaled up replica set success-200-isvc-7876a-predictor-7b564495c from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a-predictor.18c651844efd2357 namespace: kserve-ci-e2e-test resourceVersion: "27017" uid: f8554327-e91c-4c1d-892f-95dd664e029f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 8c764cae-50cd-4460-ac7a-53e47e0ead57 kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:21Z" name: success-200-isvc-7876a-predictor.18c65187cde27c70 namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: f0df101f-89d8-4d2e-95ef-45722a154674 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:59:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27010" uid: 8c764cae-50cd-4460-ac7a-53e47e0ead57 kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:21Z" name: success-200-isvc-7876a-predictor.18c65187ce4ed902 namespace: kserve-ci-e2e-test resourceVersion: "27394" uid: 12311017-8fc1-49ff-b2f7-f82e741f3c87 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:59:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27238" uid: 8c764cae-50cd-4460-ac7a-53e47e0ead57 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: success-200-isvc-7876a-predictor.18c6518ecbafcdb7 namespace: kserve-ci-e2e-test resourceVersion: "27801" uid: 866974fb-c4f3-4a4e-9fd5-7b95663a45f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:59:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7876a-predictor namespace: kserve-ci-e2e-test resourceVersion: "27238" uid: 8c764cae-50cd-4460-ac7a-53e47e0ead57 kind: Event lastTimestamp: "2026-07-28T02:00:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T02:00:06Z" name: success-200-isvc-7876a-predictor.18c6518ecc144e1f namespace: kserve-ci-e2e-test resourceVersion: "27802" uid: debf1a81-6190-4601-81fb-0b6d8dbe25b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: 31910298-692d-4c25-9f20-69c6cb517af9 kind: Event lastTimestamp: "2026-07-28T01:59:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:39Z" name: success-200-isvc-7876a.18c651844f00c4a7 namespace: kserve-ci-e2e-test resourceVersion: "27564" uid: b8dee681-8db9-49e7-b1f6-e4f74fae153e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: 31910298-692d-4c25-9f20-69c6cb517af9 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-7876a": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7876a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a.18c651844f86a7c3 namespace: kserve-ci-e2e-test resourceVersion: "27022" uid: a53d6635-a7ed-4280-a5e7-e66f23d66ffd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "26982" uid: 31910298-692d-4c25-9f20-69c6cb517af9 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7876a": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-7876a.18c651844f8701f6 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 77755b2f-efc6-4d41-bced-0a5e05b3fd12 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7876a namespace: kserve-ci-e2e-test resourceVersion: "27561" uid: 31910298-692d-4c25-9f20-69c6cb517af9 kind: Event lastTimestamp: "2026-07-28T01:59:39Z" message: InferenceService [success-200-isvc-7876a] is Ready metadata: creationTimestamp: "2026-07-28T01:59:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:39Z" name: success-200-isvc-7876a.18c6518f7f7ebd92 namespace: kserve-ci-e2e-test resourceVersion: "27563" uid: ccc59097-f646-4df5-bd3f-a219207c8466 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:48:45.638439Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f75c868390 namespace: kserve-ci-e2e-test resourceVersion: "21846" uid: e3ddac9d-adbe-49b3-a95c-72e88d125211 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-920f8-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f76712f15b namespace: kserve-ci-e2e-test resourceVersion: "21883" uid: 7e320934-6f1a-498f-ae79-7eb279906d85 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f79a84b58b namespace: kserve-ci-e2e-test resourceVersion: "21917" uid: adda2c59-a5e9-4684-b203-0498c7558779 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f79c69476f namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: d1320873-81be-45e6-a2e4-ae8c6f32511b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f79f80892b namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: 97cb318a-23af-4ded-9722-5eb912143b16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f79ffdc22e namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 7bc2c5bc-c402-41ab-9e43-a3659a15c915 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f7a0095496 namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: 61f39608-f27a-41d3-86a2-963cefce3eb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f7a2f88b1e namespace: kserve-ci-e2e-test resourceVersion: "21924" uid: 15e79ffc-8309-4756-b1ff-b561df280789 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:48:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:48:46Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f7a375406b namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 5373a285-bfb9-4c6e-a91a-96fc5ddbe879 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T01:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:58:13Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:13Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c650f7c97abc61 namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: f3618c14-9a1f-4cd9-a262-79b1474f644f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c6517ad3ba2dae namespace: kserve-ci-e2e-test resourceVersion: "26461" uid: 0f163c58-19b1-45d1-9281-d9155e20bc55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c6517ad3bace26 namespace: kserve-ci-e2e-test resourceVersion: "26464" uid: 6491f9bc-7776-455e-8b7c-54682a3ff364 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6 namespace: kserve-ci-e2e-test resourceVersion: "21842" uid: 1b0ce3af-9c29-4097-a772-2f9482b07115 kind: Event lastTimestamp: "2026-07-28T01:58:13Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:13Z" name: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6.18c6517b91ecde7b namespace: kserve-ci-e2e-test resourceVersion: "26601" uid: 06a1cb68-a76e-4280-a044-276a0a5b3777 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-920f8-predictor-585f7cfd77 namespace: kserve-ci-e2e-test resourceVersion: "21819" uid: b69cd04c-4652-455d-841b-f818b317aae6 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: 'Created pod: success-200-isvc-920f8-predictor-585f7cfd77-h5mg6' metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-920f8-predictor-585f7cfd77.18c650f75bf506b8 namespace: kserve-ci-e2e-test resourceVersion: "21843" uid: b03e6e0b-9f60-48d8-a866-20368e8d3fa8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: b59aa3c2-f605-4c71-a93e-62f1f4935726 kind: Event lastTimestamp: "2026-07-28T01:48:45Z" message: Scaled up replica set success-200-isvc-920f8-predictor-585f7cfd77 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:48:45Z" name: success-200-isvc-920f8-predictor.18c650f75918e9df namespace: kserve-ci-e2e-test resourceVersion: "21823" uid: eaf694f4-8108-4c2e-9bd3-cc54386abf06 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 62b0750c-2a75-4071-af69-c3b5d76b0089 kind: Event lastTimestamp: "2026-07-28T01:49:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:00Z" name: success-200-isvc-920f8-predictor.18c650fad78b4417 namespace: kserve-ci-e2e-test resourceVersion: "22043" uid: 68924a1b-8724-47ca-a17e-408df2466436 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "21818" uid: 62b0750c-2a75-4071-af69-c3b5d76b0089 kind: Event lastTimestamp: "2026-07-28T01:49:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:49:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:49:00Z" name: success-200-isvc-920f8-predictor.18c650fad7d4f49b namespace: kserve-ci-e2e-test resourceVersion: "22045" uid: d09d553a-2d39-4284-847b-ca589ada31bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:49:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 62b0750c-2a75-4071-af69-c3b5d76b0089 kind: Event lastTimestamp: "2026-07-28T01:50:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:15Z" name: success-200-isvc-920f8-predictor.18c650fe56f8c9e3 namespace: kserve-ci-e2e-test resourceVersion: "22744" uid: 8dc9354d-df9b-461c-8725-f211014af1e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:49:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-920f8-predictor namespace: kserve-ci-e2e-test resourceVersion: "22044" uid: 62b0750c-2a75-4071-af69-c3b5d76b0089 kind: Event lastTimestamp: "2026-07-28T01:50:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:49:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:50:15Z" name: success-200-isvc-920f8-predictor.18c650fe575cd510 namespace: kserve-ci-e2e-test resourceVersion: "22745" uid: 3fce805d-de6e-432c-8336-f4831d318210 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:48:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: fab47c5d-8d83-4dac-9535-3ef23f4f6535 kind: Event lastTimestamp: "2026-07-28T01:49:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:48:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:43Z" name: success-200-isvc-920f8.18c650f7591f303e namespace: kserve-ci-e2e-test resourceVersion: "22464" uid: eda62986-5669-48a7-84ea-133c592d90fd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-920f8 namespace: kserve-ci-e2e-test resourceVersion: "22462" uid: fab47c5d-8d83-4dac-9535-3ef23f4f6535 kind: Event lastTimestamp: "2026-07-28T01:49:43Z" message: InferenceService [success-200-isvc-920f8] is Ready metadata: creationTimestamp: "2026-07-28T01:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:49:43Z" name: success-200-isvc-920f8.18c65104d8c2619b namespace: kserve-ci-e2e-test resourceVersion: "22463" uid: 54c9027a-1bdd-4505-86e4-6129abe9836f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:57:40.783166Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26107" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-bc47b-predictor-766766c867-7h5fm to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c65173f5a31b74 namespace: kserve-ci-e2e-test resourceVersion: "26112" uid: c08af514-0537-4507-88bd-78a95d87f335 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26125" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c651740f935257 namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 83440e27-c6d4-4cd6-8153-1b04bddf47bf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c651741233d230 namespace: kserve-ci-e2e-test resourceVersion: "26173" uid: 3acc465c-e54b-42ac-bead-313a99875fa6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c6517415505792 namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: 78b6b610-82e9-4738-8dd7-21be393a7dad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c6517415c86983 namespace: kserve-ci-e2e-test resourceVersion: "26176" uid: 3bfb861f-d4c3-4c9a-97b8-cad3005f5e76 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c6517415d3899f namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: ccdeda89-cac3-4c3e-b6fc-40def6296744 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c6517418f2749e namespace: kserve-ci-e2e-test resourceVersion: "26178" uid: 08aa6685-284e-41d3-94fe-d79da9971b2a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:57:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:57:41Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c65174196bc182 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 3459717f-0712-444b-a083-cdadb127ff6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:57:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:58:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:57:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:18Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c6517428994b81 namespace: kserve-ci-e2e-test resourceVersion: "26648" uid: 20916c83-9f18-4993-865b-0d481459944a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c651844fd0f2da namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: ed6bbb92-6bf4-41b4-b02f-a8dc7b8c6d53 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:58:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:58:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:51Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c651844fd1ba4a namespace: kserve-ci-e2e-test resourceVersion: "27026" uid: 8a061220-7960-456b-8e80-70822d110d21 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-bc47b-predictor-766766c867-7h5fm namespace: kserve-ci-e2e-test resourceVersion: "26110" uid: ef7ac4cc-6535-46cc-b04a-7b376d52ef01 kind: Event lastTimestamp: "2026-07-28T01:58:53Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:58:53Z" name: success-200-isvc-bc47b-predictor-766766c867-7h5fm.18c65184ecc545b0 namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: 9f7bcc0a-f6e0-4d53-8b98-9f6487ad96c0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-bc47b-predictor-766766c867 namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 2757b531-f9a7-42c0-9962-d76c82a3194e kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'Created pod: success-200-isvc-bc47b-predictor-766766c867-7h5fm' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-bc47b-predictor-766766c867.18c65173f4e7d51b namespace: kserve-ci-e2e-test resourceVersion: "26108" uid: c3619fd5-30c2-4cdb-950b-32e3d981cd78 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26081" uid: 6d15e7ce-8cb2-4cd2-940c-76e284186902 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: Scaled up replica set success-200-isvc-bc47b-predictor-766766c867 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-bc47b-predictor.18c65173f13626de namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: b664787f-36b9-4aa5-97e1-cddc215f6beb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:57:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 7ebe9d2a-e021-47d1-a676-8349fe51e781 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-bc47b-predictor.18c651776faa3758 namespace: kserve-ci-e2e-test resourceVersion: "26544" uid: d66e7786-f6b9-4b80-9f80-bc020d3f2f81 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:57:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 7ebe9d2a-e021-47d1-a676-8349fe51e781 kind: Event lastTimestamp: "2026-07-28T01:58:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:57:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:10Z" name: success-200-isvc-bc47b-predictor.18c6517770102d3d namespace: kserve-ci-e2e-test resourceVersion: "26545" uid: 8d9af1f3-5b52-4ed8-a169-d125ecd8c03e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: 7ebe9d2a-e021-47d1-a676-8349fe51e781 kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: success-200-isvc-bc47b-predictor.18c6517e6dbcf23b namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: da74fe1c-8800-4f91-882f-a6c793528632 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T01:58:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-bc47b-predictor namespace: kserve-ci-e2e-test resourceVersion: "26317" uid: 7ebe9d2a-e021-47d1-a676-8349fe51e781 kind: Event lastTimestamp: "2026-07-28T01:58:40Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:58:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:58:40Z" name: success-200-isvc-bc47b-predictor.18c6517e6e25ccd9 namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: d3e06451-8b4c-42a6-b7cb-42d0511aefde reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: eb2368dd-7384-43cf-be82-d7cdb2c1ee75 kind: Event lastTimestamp: "2026-07-28T01:58:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:28Z" name: success-200-isvc-bc47b.18c65173f1592211 namespace: kserve-ci-e2e-test resourceVersion: "26754" uid: 5683f0d7-8738-4349-ae35-f6b3d67ce5a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: eb2368dd-7384-43cf-be82-d7cdb2c1ee75 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'Failed to update status for InferenceService "success-200-isvc-bc47b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bc47b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-bc47b.18c65173f2a598b7 namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: a5634fa5-98fd-4649-85af-6e73fa6be400 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:57:40Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: eb2368dd-7384-43cf-be82-d7cdb2c1ee75 kind: Event lastTimestamp: "2026-07-28T01:57:40Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-bc47b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:57:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:57:40Z" name: success-200-isvc-bc47b.18c65173f2a5e95b namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: 4b3bc8f1-92c7-477f-8d7e-28a5fabb14fb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:58:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-bc47b namespace: kserve-ci-e2e-test resourceVersion: "26752" uid: eb2368dd-7384-43cf-be82-d7cdb2c1ee75 kind: Event lastTimestamp: "2026-07-28T01:58:28Z" message: InferenceService [success-200-isvc-bc47b] is Ready metadata: creationTimestamp: "2026-07-28T01:58:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:58:28Z" name: success-200-isvc-bc47b.18c6517f20940587 namespace: kserve-ci-e2e-test resourceVersion: "26753" uid: 33f61692-1630-4fb1-9e11-f88caacd5b9c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:35:50.284054Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14787" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c7854-predictor-644bcbbbf7-7k64h to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c65042d5cd9ced namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: fc194721-5efc-49d1-8144-8f0f2b028119 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14798" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Add eth0 [10.134.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c65042f0afaf2e namespace: kserve-ci-e2e-test resourceVersion: "14874" uid: 1b09f3ce-79c8-4c9e-8ed2-915647d0f925 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c65042f231aaa5 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: a58e2fa8-bfb8-44df-b302-74df4b1e7e5e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" in 17.423s (17.423s including waiting). Image size: 1334926449 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6504700b3efbb namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 7f2fb85e-e91a-45fc-a3f1-73b522ebaddf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c65047048b539c namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 4094d1ea-2d86-4f51-9fb7-419c75d8d5fd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c650470505a78c namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: a3be737d-4528-43de-acc1-98c0e09834a1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:08Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T01:36:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:08Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6504705759e44 namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: a2953d63-1cc2-433f-a9b5-ade65012f275 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.514s (4.514s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c650481286ddb0 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: c01e09ea-5151-4e47-be53-31ee144bde53 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c650481943857f namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: 677da772-5ace-43a5-a99f-f9ef67b24863 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:36:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:12Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:36:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:12Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6504819efcd17 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: 31dab205-b9cc-4ef9-98d9-308439429f18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T01:36:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:36:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:36:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:36:59Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c65048334100af namespace: kserve-ci-e2e-test resourceVersion: "15597" uid: 2127d97c-870d-433f-9170-4e0b1a096706 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6505a3737a21c namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: aab9b19c-9083-42c2-a84c-0457aed658fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6505a373815bf namespace: kserve-ci-e2e-test resourceVersion: "15943" uid: 967c4074-e6cf-4753-91f5-797fd85c11b7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h namespace: kserve-ci-e2e-test resourceVersion: "14790" uid: 5bdec550-0d4b-453e-ac45-84fea7a86bd2 kind: Event lastTimestamp: "2026-07-28T01:37:34Z" message: 'Readiness probe failed: Get "https://10.134.0.21:8643/healthz": dial tcp 10.134.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T01:37:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:34Z" name: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h.18c6505b0f79d55e namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 60900c66-ee6a-4c21-af47-42dcdb8d0115 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c7854-predictor-644bcbbbf7 namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: 0abc3757-dd90-429a-8c0f-5f96b61ba4b4 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Created pod: success-200-isvc-c7854-predictor-644bcbbbf7-7k64h' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854-predictor-644bcbbbf7.18c65042d563d801 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: 341e379e-d446-41b4-8755-29277b7c4b56 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 5a696a3a-46e1-4521-a78e-ee43a5078d30 kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: Scaled up replica set success-200-isvc-c7854-predictor-644bcbbbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854-predictor.18c65042d10a9bfe namespace: kserve-ci-e2e-test resourceVersion: "14784" uid: de65d1f6-c81f-4881-bcf9-58d89d839efa reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: dc9cd2c1-6ac0-4b17-958e-52ae4b5ef0c6 kind: Event lastTimestamp: "2026-07-28T01:36:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:50Z" name: success-200-isvc-c7854-predictor.18c6504656df8937 namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 8bca0053-52e1-40b4-9162-c55e306c38bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "14795" uid: dc9cd2c1-6ac0-4b17-958e-52ae4b5ef0c6 kind: Event lastTimestamp: "2026-07-28T01:36:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:36:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:36:50Z" name: success-200-isvc-c7854-predictor.18c650465726ce12 namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: b15b312a-ec9a-4b1b-8dc0-e19ddf154a02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: dc9cd2c1-6ac0-4b17-958e-52ae4b5ef0c6 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: success-200-isvc-c7854-predictor.18c6504d54e94dc3 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 5f6cdac6-a85d-4516-8170-f0bae740ff8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T01:36:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7854-predictor namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: dc9cd2c1-6ac0-4b17-958e-52ae4b5ef0c6 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:36:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: success-200-isvc-c7854-predictor.18c6504d5551c97f namespace: kserve-ci-e2e-test resourceVersion: "15796" uid: bed5bab9-f1ee-4e66-b626-c54c9ade67f5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 10730df5-9516-4b4d-a1bc-6aa6f12d485f kind: Event lastTimestamp: "2026-07-28T01:37:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:09Z" name: success-200-isvc-c7854.18c65042d72fdd29 namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: d7bf68db-bd79-4454-9b9d-e19a8bb23db9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 10730df5-9516-4b4d-a1bc-6aa6f12d485f kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'Failed to update status for InferenceService "success-200-isvc-c7854": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7854": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854.18c65042d79d1cb1 namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 7a8d7021-fc3f-4db1-9161-101441c2e253 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:35:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "14773" uid: 10730df5-9516-4b4d-a1bc-6aa6f12d485f kind: Event lastTimestamp: "2026-07-28T01:35:50Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7854": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T01:35:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:35:50Z" name: success-200-isvc-c7854.18c65042d79d7f4b namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: 9a9431fa-2584-4df1-a168-6741b234893a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7854 namespace: kserve-ci-e2e-test resourceVersion: "15685" uid: 10730df5-9516-4b4d-a1bc-6aa6f12d485f kind: Event lastTimestamp: "2026-07-28T01:37:09Z" message: InferenceService [success-200-isvc-c7854] is Ready metadata: creationTimestamp: "2026-07-28T01:37:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:09Z" name: success-200-isvc-c7854.18c6505542503374 namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 9cd2de43-db40-42ed-ae9f-09e07dde26dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:38:40.796387Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-33843-84bc469c8d-t78gn to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:38:40Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:38:40Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506a89214314 namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: d75fcf7a-ed8b-4660-8dce-27274ed876d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:40Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-33843-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:40Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506a8fad0980 namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: fa403e10-c9dd-4611-bcf3-5346627cf63d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16891" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:38:41Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:38:41Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506ab4aaaa4a namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 3d91351c-a4aa-4bc7-aebb-2cc8d1bd4e7b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-33843} kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:38:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:41Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506ab60f5ccd namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 7401060b-be4d-4bfc-8bdd-3a6aa331a2b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-33843} kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:38:41Z" message: 'Created container: switch-graph-33843' metadata: creationTimestamp: "2026-07-28T01:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:41Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506ab8f7d8a3 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: bc8ba47f-4622-41e2-98e0-8ce7fb4b82a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-33843} kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:38:41Z" message: Started container switch-graph-33843 metadata: creationTimestamp: "2026-07-28T01:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:41Z" name: switch-graph-33843-84bc469c8d-t78gn.18c6506ab96b5617 namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 3ac045c0-e700-4562-b99c-a3066c63176c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:46:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-33843} kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:46:55Z" message: Stopping container switch-graph-33843 metadata: creationTimestamp: "2026-07-28T01:46:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:46:55Z" name: switch-graph-33843-84bc469c8d-t78gn.18c650ddb0b216b7 namespace: kserve-ci-e2e-test resourceVersion: "20372" uid: 9765860b-d33e-4479-b0e1-53e1fa48449e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:46:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-33843} kind: Pod name: switch-graph-33843-84bc469c8d-t78gn namespace: kserve-ci-e2e-test resourceVersion: "16885" uid: 0d8ae9df-549f-41ea-9e81-7860ac4a4cb8 kind: Event lastTimestamp: "2026-07-28T01:47:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:46:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:47:23Z" name: switch-graph-33843-84bc469c8d-t78gn.18c650de6efb2cc1 namespace: kserve-ci-e2e-test resourceVersion: "20718" uid: fbdf758d-9722-4025-9881-b73aa3b5e9a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:40Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-33843-84bc469c8d namespace: kserve-ci-e2e-test resourceVersion: "16879" uid: fe461126-c86e-43a9-b832-90a45e106d2f kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: 'Created pod: switch-graph-33843-84bc469c8d-t78gn' metadata: creationTimestamp: "2026-07-28T01:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: switch-graph-33843-84bc469c8d.18c6506a88c03365 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: 054fb8ec-28b8-4763-8d7a-e326b4ea625f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:40Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "16877" uid: 4d988ffb-3913-474b-8c69-fd358a2f0c56 kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: Scaled up replica set switch-graph-33843-84bc469c8d from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:40Z" name: switch-graph-33843.18c6506a87bb0cd4 namespace: kserve-ci-e2e-test resourceVersion: "16880" uid: f0f9bf2c-91b2-4fbf-845d-f867abceedb2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:40Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: 712ebc85-74d9-460e-afc1-023df073eb5f kind: Event lastTimestamp: "2026-07-28T01:38:40Z" message: InferenceGraph [switch-graph-33843] is Ready metadata: creationTimestamp: "2026-07-28T01:38:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:38:40Z" name: switch-graph-33843.18c6506a8cce4e8b namespace: kserve-ci-e2e-test resourceVersion: "16895" uid: f427d9b5-002c-4dc4-866f-15c811f9dbfc reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 9009a2dd-727c-440c-8c89-47054890a525 kind: Event lastTimestamp: "2026-07-28T01:38:55Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:55Z" name: switch-graph-33843.18c6506e061dcc84 namespace: kserve-ci-e2e-test resourceVersion: "17024" uid: e88456da-0ef6-45ba-b128-48e30f865760 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:38:55Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "16878" uid: 9009a2dd-727c-440c-8c89-47054890a525 kind: Event lastTimestamp: "2026-07-28T01:38:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:38:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:38:55Z" name: switch-graph-33843.18c6506e0685dbc4 namespace: kserve-ci-e2e-test resourceVersion: "17026" uid: 7ad6a4f8-18e3-403c-9ef7-ed9d2717c555 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 9009a2dd-727c-440c-8c89-47054890a525 kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: switch-graph-33843.18c65071852d0f2e namespace: kserve-ci-e2e-test resourceVersion: "17183" uid: 83564bc0-3774-4557-a3df-1a4120a78b71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:39:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-33843 namespace: kserve-ci-e2e-test resourceVersion: "17025" uid: 9009a2dd-727c-440c-8c89-47054890a525 kind: Event lastTimestamp: "2026-07-28T01:39:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:39:10Z" name: switch-graph-33843.18c65071858f9085 namespace: kserve-ci-e2e-test resourceVersion: "17185" uid: ebbd71c8-a62a-45e4-a746-c758c199a2fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:59:20.435255Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27369" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-704b9-699d45d87-vr6gx to ip-10-0-134-120.ec2.internal metadata: creationTimestamp: "2026-07-28T01:59:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:59:20Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b295d4530 namespace: kserve-ci-e2e-test resourceVersion: "27375" uid: 7a1a1c28-62d8-4870-a279-0c2f3e254af6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T01:59:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-704b9-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:20Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b35b0241a namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: 2ee784a3-f878-414c-9a9a-c358928e3e54 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:59:21Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b66fe732f namespace: kserve-ci-e2e-test resourceVersion: "27402" uid: 9d1ecc0f-3874-40ef-903a-a749188452c0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-704b9} kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" already present on machine metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:21Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b682df40e namespace: kserve-ci-e2e-test resourceVersion: "27404" uid: 7dbd172c-aa66-497d-a031-5d11f9777a54 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-704b9} kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: 'Created container: switch-graph-704b9' metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:21Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b6b1d7a45 namespace: kserve-ci-e2e-test resourceVersion: "27405" uid: 5b706e9e-dbfd-44a0-9af0-8b0a7f6e61fa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-704b9} kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T01:59:21Z" message: Started container switch-graph-704b9 metadata: creationTimestamp: "2026-07-28T01:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:59:21Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6518b6b95503c namespace: kserve-ci-e2e-test resourceVersion: "27406" uid: ff083835-9449-4f11-9e62-829a409a1f70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T02:15:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-704b9} kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T02:15:39Z" message: Stopping container switch-graph-704b9 metadata: creationTimestamp: "2026-07-28T02:15:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:15:39Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6526f1e507828 namespace: kserve-ci-e2e-test resourceVersion: "34272" uid: 75beaf36-a177-4e2e-accb-24a299b711c1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T02:15:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-704b9} kind: Pod name: switch-graph-704b9-699d45d87-vr6gx namespace: kserve-ci-e2e-test resourceVersion: "27372" uid: 81f7fa45-630c-432c-bc84-95bdfeabce3b kind: Event lastTimestamp: "2026-07-28T02:16:03Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T02:15:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T02:16:03Z" name: switch-graph-704b9-699d45d87-vr6gx.18c6526ff4351c24 namespace: kserve-ci-e2e-test resourceVersion: "34515" uid: 594caf10-bbe1-41f0-abc4-1c58e4cd0e1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-120.ec2.internal source: component: kubelet host: ip-10-0-134-120.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-704b9-699d45d87 namespace: kserve-ci-e2e-test resourceVersion: "27366" uid: 5f7abb8c-478c-442f-a64c-5e2ce24315f8 kind: Event lastTimestamp: "2026-07-28T01:59:20Z" message: 'Created pod: switch-graph-704b9-699d45d87-vr6gx' metadata: creationTimestamp: "2026-07-28T01:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:20Z" name: switch-graph-704b9-699d45d87.18c6518b287ffd4c namespace: kserve-ci-e2e-test resourceVersion: "27370" uid: d726325c-66a7-47fd-8d34-06e2d544ae94 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27364" uid: fc82fcc0-6ed6-40d3-b7e2-21814c14e2f6 kind: Event lastTimestamp: "2026-07-28T01:59:20Z" message: Scaled up replica set switch-graph-704b9-699d45d87 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:20Z" name: switch-graph-704b9.18c6518b2691ba2b namespace: kserve-ci-e2e-test resourceVersion: "27367" uid: f88946b9-09bd-4aa7-858e-fcd9d56777c2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: f9ac3bb6-b152-4659-ba28-141417191cd9 kind: Event lastTimestamp: "2026-07-28T01:59:20Z" message: InferenceGraph [switch-graph-704b9] is Ready metadata: creationTimestamp: "2026-07-28T01:59:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:59:20Z" name: switch-graph-704b9.18c6518b2e172a5f namespace: kserve-ci-e2e-test resourceVersion: "27383" uid: 22bfdba5-710d-41b2-a7c8-1019d8c4f18e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: ac508fa1-7b51-4632-877e-b22e6bfa13c0 kind: Event lastTimestamp: "2026-07-28T01:59:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:35Z" name: switch-graph-704b9.18c6518ea5d81bfe namespace: kserve-ci-e2e-test resourceVersion: "27515" uid: 5a71a84e-9b91-441c-91ad-1b06b0888d5e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: ac508fa1-7b51-4632-877e-b22e6bfa13c0 kind: Event lastTimestamp: "2026-07-28T01:59:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T01:59:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:35Z" name: switch-graph-704b9.18c6518ea63cdffc namespace: kserve-ci-e2e-test resourceVersion: "27517" uid: 3fb5771f-cd02-48ca-a0a0-39daeb5cf48c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27516" uid: ac508fa1-7b51-4632-877e-b22e6bfa13c0 kind: Event lastTimestamp: "2026-07-28T01:59:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:50Z" name: switch-graph-704b9.18c6519224edc14e namespace: kserve-ci-e2e-test resourceVersion: "27643" uid: db4e40f3-85ab-40c2-8af2-126559ccc201 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:59:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-704b9 namespace: kserve-ci-e2e-test resourceVersion: "27516" uid: ac508fa1-7b51-4632-877e-b22e6bfa13c0 kind: Event lastTimestamp: "2026-07-28T01:59:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T01:59:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:59:50Z" name: switch-graph-704b9.18c651922558478d namespace: kserve-ci-e2e-test resourceVersion: "27644" uid: 3f9852ea-cd6a-4a6c-b5cd-f06ed8b6554d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T01:37:20.347745Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15779" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c7854-5f88778cb9-czp4v to ip-10-0-142-30.ec2.internal metadata: creationTimestamp: "2026-07-28T01:37:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T01:37:20Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c65057ce0465d5 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: ff81a87e-4890-4655-bf03-5e01e11de8c1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5dff678656-txz5t source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:20Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-c7854-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T01:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:20Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c65057d8d7325e namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: f7c4424c-4768-48ac-8adf-3888f9394c02 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15789" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:21Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T01:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T01:37:21Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c650580b21ed14 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 60c2ebb7-885b-49b1-a055-46ce5d8e5523 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:21Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" metadata: creationTimestamp: "2026-07-28T01:37:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:21Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c650580c60fc1a namespace: kserve-ci-e2e-test resourceVersion: "15814" uid: 5160d2dc-0a81-4d15-9b84-864734f465c0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:3f8b4d8bbc07de8aec3f29b089d69c216cddd2ca68e10f109f830f4b0fdd9ee1" in 1.899s (1.899s including waiting). Image size: 218232445 bytes.' metadata: creationTimestamp: "2026-07-28T01:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:23Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c650587d94eef8 namespace: kserve-ci-e2e-test resourceVersion: "15828" uid: 585013b5-a9d4-4a77-b6ea-e7699edcbc70 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:23Z" message: 'Created container: switch-graph-c7854' metadata: creationTimestamp: "2026-07-28T01:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:23Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c650588046f954 namespace: kserve-ci-e2e-test resourceVersion: "15829" uid: b7cff29b-3c34-4a13-abc4-6294e0c2b629 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:23Z" message: Started container switch-graph-c7854 metadata: creationTimestamp: "2026-07-28T01:37:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:23Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c6505880a78153 namespace: kserve-ci-e2e-test resourceVersion: "15830" uid: a04f97d0-c400-410c-9335-d72f0e051be6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:37:30Z" message: Stopping container switch-graph-c7854 metadata: creationTimestamp: "2026-07-28T01:37:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:37:30Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c6505a2e77f558 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 79531ab6-64ff-40a2-8ede-316c5cc56e82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T01:37:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c7854} kind: Pod name: switch-graph-c7854-5f88778cb9-czp4v namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: adfe9417-6bd9-47e1-b96b-ae5f82f63dea kind: Event lastTimestamp: "2026-07-28T01:38:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T01:37:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T01:38:00Z" name: switch-graph-c7854-5f88778cb9-czp4v.18c6505b43c16866 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 5e12969f-7fe2-4cb3-ada5-bdbe5cdeac0e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-30.ec2.internal source: component: kubelet host: ip-10-0-142-30.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c7854-5f88778cb9 namespace: kserve-ci-e2e-test resourceVersion: "15776" uid: 39604c96-40e2-49db-b62a-7e7a3752f7f5 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: 'Created pod: switch-graph-c7854-5f88778cb9-czp4v' metadata: creationTimestamp: "2026-07-28T01:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: switch-graph-c7854-5f88778cb9.18c65057cd9326a5 namespace: kserve-ci-e2e-test resourceVersion: "15780" uid: a6cb9e13-000b-4ba7-9363-ad9f57a75061 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c7854 namespace: kserve-ci-e2e-test resourceVersion: "15773" uid: d1a85cdf-0e1d-4845-9cb5-4bdef54a4152 kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: Scaled up replica set switch-graph-c7854-5f88778cb9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T01:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T01:37:20Z" name: switch-graph-c7854.18c65057ccae6dbe namespace: kserve-ci-e2e-test resourceVersion: "15777" uid: 612c31df-2ee6-4d49-b0b2-1050f58980ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T01:37:20Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c7854 namespace: kserve-ci-e2e-test resourceVersion: "15792" uid: 0ddff108-95c6-4252-9cba-4e4744062a9a kind: Event lastTimestamp: "2026-07-28T01:37:20Z" message: InferenceGraph [switch-graph-c7854] is Ready metadata: creationTimestamp: "2026-07-28T01:37:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T01:37:20Z" name: switch-graph-c7854.18c65057d1dbfbea namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: 61d1cba7-aeea-4207-a8b3-a51e4ee17a60 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "34549"