--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:12:33.693264Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-68f71-7fd665b646-ccjl8 to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:12:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:12:33Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef3893db77ac namespace: kserve-ci-e2e-test resourceVersion: "20933" uid: a78ffe3f-e6e3-441b-bf56-fbc7942325cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:34Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:12:34Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:12:34Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef38ad59c200 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: 811abca2-7947-4660-9ec0-8c7b7f3d9180 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68f71} kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:12:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:34Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef38aef3d4a2 namespace: kserve-ci-e2e-test resourceVersion: "20951" uid: 52ef53f9-ea06-4743-91c0-a03f190114eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68f71} kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:12:34Z" message: 'Created container: ensemble-graph-68f71' metadata: creationTimestamp: "2026-07-13T19:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:34Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef38b23756b1 namespace: kserve-ci-e2e-test resourceVersion: "20952" uid: 04756a72-83d3-46de-9f47-14d5bc8389dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68f71} kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:12:34Z" message: Started container ensemble-graph-68f71 metadata: creationTimestamp: "2026-07-13T19:12:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:34Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef38b2a3cdb8 namespace: kserve-ci-e2e-test resourceVersion: "20953" uid: 4f69bfbe-08c4-4c22-a702-d5d129854c0b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68f71} kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Stopping container ensemble-graph-68f71 metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:43Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef3aebd6e364 namespace: kserve-ci-e2e-test resourceVersion: "21045" uid: 35322a05-c1f4-432e-9eef-7bf32550b3ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-68f71} kind: Pod name: ensemble-graph-68f71-7fd665b646-ccjl8 namespace: kserve-ci-e2e-test resourceVersion: "20930" uid: e7baffbb-1ff1-4105-8ab8-5a9b7464f0cb kind: Event lastTimestamp: "2026-07-13T19:13:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:10Z" name: ensemble-graph-68f71-7fd665b646-ccjl8.18c1ef3b6cbb7e7a namespace: kserve-ci-e2e-test resourceVersion: "21438" uid: bf018ad3-7858-4ab6-b42d-85383ded694f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-68f71-7fd665b646 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: 9a8dbb4c-cee1-4fb9-8e6c-1805a1ae0b3c kind: Event lastTimestamp: "2026-07-13T19:12:33Z" message: 'Created pod: ensemble-graph-68f71-7fd665b646-ccjl8' metadata: creationTimestamp: "2026-07-13T19:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:33Z" name: ensemble-graph-68f71-7fd665b646.18c1ef3893660e59 namespace: kserve-ci-e2e-test resourceVersion: "20929" uid: 0113af0c-6983-4034-810f-5da7c1673f6f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: 868c9006-a1ca-4b02-927a-4d9367722f8a kind: Event lastTimestamp: "2026-07-13T19:12:33Z" message: Scaled up replica set ensemble-graph-68f71-7fd665b646 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:33Z" name: ensemble-graph-68f71.18c1ef389274cbea namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: af8d1964-0b11-4aa9-978f-e9dba1c04352 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20939" uid: 44ecf3c2-0f47-432e-ad4c-a0f2bbc565f8 kind: Event lastTimestamp: "2026-07-13T19:12:33Z" message: InferenceGraph [ensemble-graph-68f71] is Ready metadata: creationTimestamp: "2026-07-13T19:12:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:33Z" name: ensemble-graph-68f71.18c1ef3897d441c1 namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: 33c1d5d3-4cfe-46d0-9edc-53e8d18c243e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:13:54.021969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21973" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-ef1db-6b8465b4f9-cdj45 to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1ef4b47d241c0 namespace: kserve-ci-e2e-test resourceVersion: "21978" uid: 97ea4855-5fd3-4025-b9da-3914d9f7ba30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1ef4b61b8bf37 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 55059707-6c78-40d7-8ac9-ed836743ec68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ef1db} kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1ef4b62ff6d57 namespace: kserve-ci-e2e-test resourceVersion: "21994" uid: bbd04745-46af-4012-8b25-e5dac45177cd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ef1db} kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: 'Created container: ensemble-graph-ef1db' metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1ef4b65dcf1a4 namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: c95f0e56-fe09-44d8-a683-26105cc532f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ef1db} kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: Started container ensemble-graph-ef1db metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1ef4b66b2288c namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 1bb80b56-787f-4742-80dc-fa8246a30a74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ef1db} kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: Stopping container ensemble-graph-ef1db metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:08Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1efbe75a03a62 namespace: kserve-ci-e2e-test resourceVersion: "25764" uid: 8fe80620-d8ae-4cae-9632-71d0fbca655c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-ef1db} kind: Pod name: ensemble-graph-ef1db-6b8465b4f9-cdj45 namespace: kserve-ci-e2e-test resourceVersion: "21975" uid: 5366557b-9c65-4f41-baf7-3180fa856108 kind: Event lastTimestamp: "2026-07-13T19:22:36Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:36Z" name: ensemble-graph-ef1db-6b8465b4f9-cdj45.18c1efbf0e1aa310 namespace: kserve-ci-e2e-test resourceVersion: "26105" uid: 7f53dc6a-733c-4872-959b-5f4e362e4db6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-ef1db-6b8465b4f9 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: f7afacbf-b605-46b9-8d7e-5989e9e7c655 kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: 'Created pod: ensemble-graph-ef1db-6b8465b4f9-cdj45' metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db-6b8465b4f9.18c1ef4b474ca6ee namespace: kserve-ci-e2e-test resourceVersion: "21974" uid: 873ec2e5-00e6-4675-91e6-b165fc12a2bd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21968" uid: aa438fd9-e72b-4175-9a6d-8e45ab6770cb kind: Event lastTimestamp: "2026-07-13T19:13:53Z" message: Scaled up replica set ensemble-graph-ef1db-6b8465b4f9 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db.18c1ef4b467c8c6e namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: eb929502-da02-4d85-b476-434b97e7f338 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21986" uid: 7c8358f6-9a8a-419d-8d95-5b2d75817d0b kind: Event lastTimestamp: "2026-07-13T19:13:54Z" message: InferenceGraph [ensemble-graph-ef1db] is Ready metadata: creationTimestamp: "2026-07-13T19:13:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:54Z" name: ensemble-graph-ef1db.18c1ef4b4bef2cf6 namespace: kserve-ci-e2e-test resourceVersion: "21988" uid: 40b58c60-de56-4b43-9c73-643a0505c0ad reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: dd05f12d-e8da-4c23-9ddd-6a236aa0f6e0 kind: Event lastTimestamp: "2026-07-13T19:14:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:24Z" name: ensemble-graph-ef1db.18c1ef4ec510a591 namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: 72b85f76-32d2-4ec1-87ac-c2b921cb1699 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:14:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: dd05f12d-e8da-4c23-9ddd-6a236aa0f6e0 kind: Event lastTimestamp: "2026-07-13T19:14:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:14:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:24Z" name: ensemble-graph-ef1db.18c1ef4ec562617f namespace: kserve-ci-e2e-test resourceVersion: "22245" uid: 7716d80e-6926-4b08-a11c-9f7ac9b54936 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:23:29.340221Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26821" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-08823-predictor-698559b985-8l49b to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd13b7622c8 namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: f9cd96cc-2731-4d71-a1a3-de2f27a0a5da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26829" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd156b6abe5 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 80fcb6d5-7672-46d8-8506-0ae13e2077ea reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd158143a8e namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: a0cb78b5-a37e-457f-afbf-4d6bffa4330e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd15acc3a45 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 265a042b-a41f-40c3-a8c5-5db5167110da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd15b38d9ae namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 367f5ada-a393-4547-ab30-396865c56549 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd15b43cda2 namespace: kserve-ci-e2e-test resourceVersion: "26856" uid: 7c78349b-4e28-4e63-9e04-6a369783adb5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd15e0eb2c4 namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: 4ee230a3-4c7f-4bcd-9928-2f0013c66040 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd15e86ca92 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: 02e5c3e8-b538-44d8-bed0-defc675ce383 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:32:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:56Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1efd16d8b7689 namespace: kserve-ci-e2e-test resourceVersion: "31021" uid: 65c4c9e8-b109-4fe4-a842-ff79c99c7d90 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:32:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:54Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1f054ba82bb6d namespace: kserve-ci-e2e-test resourceVersion: "31002" uid: 6ee31d1e-9f63-4738-87a3-6c62d795edcb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:32:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:54Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1f054ba83376a namespace: kserve-ci-e2e-test resourceVersion: "31003" uid: f8decab7-a17e-41a8-b818-0244808ba43f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-08823-predictor-698559b985-8l49b namespace: kserve-ci-e2e-test resourceVersion: "26823" uid: b07f8e20-debd-401e-9683-3962605d94e5 kind: Event lastTimestamp: "2026-07-13T19:32:56Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:56Z" name: error-404-isvc-08823-predictor-698559b985-8l49b.18c1f055365d806d namespace: kserve-ci-e2e-test resourceVersion: "31020" uid: 8ec250d7-49b4-4c39-a28e-a9e468b744e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-08823-predictor-698559b985 namespace: kserve-ci-e2e-test resourceVersion: "26814" uid: 58c7c8b0-2828-498d-9185-6436945739e9 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Created pod: error-404-isvc-08823-predictor-698559b985-8l49b' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor-698559b985.18c1efd13af2f04a namespace: kserve-ci-e2e-test resourceVersion: "26822" uid: 8ab0b3cb-4e2c-4b6a-ac9e-53f2864f1708 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26812" uid: b446a750-6436-40cf-aeeb-da6450e5d549 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Scaled up replica set error-404-isvc-08823-predictor-698559b985 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823-predictor.18c1efd138ffb102 namespace: kserve-ci-e2e-test resourceVersion: "26815" uid: 265c0478-7b53-4ee4-800a-4c4404cc7577 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:23:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 44f0db17-61e2-4987-bdf7-a4aef6fbc1fa kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:23:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: error-404-isvc-08823-predictor.18c1efd4b782dcbf namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: 724ce46d-a128-4e30-a4b9-b877706d610f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:23:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26813" uid: 44f0db17-61e2-4987-bdf7-a4aef6fbc1fa kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:23:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: error-404-isvc-08823-predictor.18c1efd4b7e9361c namespace: kserve-ci-e2e-test resourceVersion: "27176" uid: 9b6186cd-1b40-42bc-88ac-c2127feab490 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:24:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 44f0db17-61e2-4987-bdf7-a4aef6fbc1fa kind: Event lastTimestamp: "2026-07-13T19:24:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:44Z" name: error-404-isvc-08823-predictor.18c1efdbb54a24ca namespace: kserve-ci-e2e-test resourceVersion: "27569" uid: 6a04d77c-08c7-49d9-a4b3-a9879e7b2ee7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:24:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 44f0db17-61e2-4987-bdf7-a4aef6fbc1fa kind: Event lastTimestamp: "2026-07-13T19:24:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:44Z" name: error-404-isvc-08823-predictor.18c1efdbb5ac0ebb namespace: kserve-ci-e2e-test resourceVersion: "27570" uid: 2c1fd90a-6306-4c42-9e96-f1a9785c90c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: 492003e3-eb11-4fc6-8de9-f7fc0ecdbcef kind: Event lastTimestamp: "2026-07-13T19:24:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:24:26Z" name: error-404-isvc-08823.18c1efd13913d7c7 namespace: kserve-ci-e2e-test resourceVersion: "27395" uid: b8c380d9-eea8-4eb2-81e6-f1fb95f2801f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: 492003e3-eb11-4fc6-8de9-f7fc0ecdbcef kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Failed to update status for InferenceService "error-404-isvc-08823": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-08823": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823.18c1efd139bfb6a1 namespace: kserve-ci-e2e-test resourceVersion: "26818" uid: 4981ba53-8c26-491b-ac21-4c82498b7a19 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26799" uid: 492003e3-eb11-4fc6-8de9-f7fc0ecdbcef kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-08823": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-08823.18c1efd139bffe3c namespace: kserve-ci-e2e-test resourceVersion: "26819" uid: 81c95076-7879-43d3-b2f5-4bc745c17150 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "27392" uid: 492003e3-eb11-4fc6-8de9-f7fc0ecdbcef kind: Event lastTimestamp: "2026-07-13T19:24:26Z" message: InferenceService [error-404-isvc-08823] is Ready metadata: creationTimestamp: "2026-07-13T19:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:24:26Z" name: error-404-isvc-08823.18c1efde7d8f2e87 namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: 9300e8de-df40-47ee-98fa-6f378d9f1c48 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:00:25.037340Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-19366-predictor-758f8cb57-swfg9 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:00:25Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee8eec947912 namespace: kserve-ci-e2e-test resourceVersion: "14626" uid: 159314cc-d1b2-4f7d-b7c1-1dd3428fa353 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-19366-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee8ef3a4870e namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: 51fa7434-1aaf-4d6d-9cdb-dd1c359ab2e1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: Add eth0 [10.134.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:00:26Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee8f2b0b743d namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 0bac5209-c973-4796-8a99-c8ffb5fc1450 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:26Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee8f2c6b3117 namespace: kserve-ci-e2e-test resourceVersion: "14697" uid: 25284c74-a624-47a8-803e-ea6661155d81 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" in 15.316s (15.316s including waiting). Image size: 1334700633 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee92bd5b8d81 namespace: kserve-ci-e2e-test resourceVersion: "14858" uid: cc680d2d-541c-47d1-98f0-3f46ea5ca4fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee92c0d69ffa namespace: kserve-ci-e2e-test resourceVersion: "14859" uid: b2848204-dfa1-41d7-be16-b08e3eda895b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee92c1784ea3 namespace: kserve-ci-e2e-test resourceVersion: "14862" uid: cdbe2cba-4101-4e12-8d4d-93137e6cdd79 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee92c185bd9b namespace: kserve-ci-e2e-test resourceVersion: "14866" uid: d0f6459c-995e-4673-b7fb-455dd5822539 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.551s (2.551s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee9359a118e9 namespace: kserve-ci-e2e-test resourceVersion: "14882" uid: c2d4a909-03a5-4226-b914-a24489480320 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee935eb65ed3 namespace: kserve-ci-e2e-test resourceVersion: "14884" uid: 99280261-915e-4a0a-8416-b288e640099c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee935f2cb892 namespace: kserve-ci-e2e-test resourceVersion: "14886" uid: 8633fd6f-1f5f-4b80-9f23-24bfb2c7f240 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:00:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:02:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:00:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:01Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1ee93be6edb67 namespace: kserve-ci-e2e-test resourceVersion: "15761" uid: 63533b5f-db3f-4319-97ce-a651b2a428ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1eea4bd2dca0d namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 7b82be10-1ab7-4300-a291-024dc5b5fbb9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1eea4bd313103 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 16608820-a3e6-48fb-8f42-030c6d4d7cea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-19366-predictor-758f8cb57-swfg9 namespace: kserve-ci-e2e-test resourceVersion: "14623" uid: 9bf5f463-4192-4760-8fff-33b41a409c0e kind: Event lastTimestamp: "2026-07-13T19:02:01Z" message: 'Readiness probe failed: Get "https://10.134.0.23:8643/healthz": dial tcp 10.134.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:01Z" name: error-404-isvc-19366-predictor-758f8cb57-swfg9.18c1eea57088f803 namespace: kserve-ci-e2e-test resourceVersion: "15760" uid: 71183ec2-d8ce-4a6a-a003-0d8ccaf95e07 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-19366-predictor-758f8cb57 namespace: kserve-ci-e2e-test resourceVersion: "14615" uid: 927a65d5-035f-4e96-afd6-6b94b42a5c70 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: 'Created pod: error-404-isvc-19366-predictor-758f8cb57-swfg9' metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:25Z" name: error-404-isvc-19366-predictor-758f8cb57.18c1ee8eebfb2794 namespace: kserve-ci-e2e-test resourceVersion: "14622" uid: 9741ca44-eec3-4ad4-b23d-641b5da5b1b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: ec56000c-169c-4318-abdc-aa33ec6155f1 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: Scaled up replica set error-404-isvc-19366-predictor-758f8cb57 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: error-404-isvc-19366-predictor.18c1ee8eea291334 namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 2f9798a3-8a97-41c7-ae04-61d56677b2ea reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 3a4d21ee-2b90-4af0-b885-9abd3dace114 kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:10Z" name: error-404-isvc-19366-predictor.18c1ee9268c0fcf4 namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: 99e211b7-66c5-4321-b422-c8b24ed20db5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: 3a4d21ee-2b90-4af0-b885-9abd3dace114 kind: Event lastTimestamp: "2026-07-13T19:01:10Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:10Z" name: error-404-isvc-19366-predictor.18c1ee92690c1b9b namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 338bc97d-a04b-4465-bb8f-3e3eecb640bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 3a4d21ee-2b90-4af0-b885-9abd3dace114 kind: Event lastTimestamp: "2026-07-13T19:01:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:55Z" name: error-404-isvc-19366-predictor.18c1ee9ce5588073 namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: a6881a5a-4062-4942-aa80-7c92a1369e9b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14842" uid: 3a4d21ee-2b90-4af0-b885-9abd3dace114 kind: Event lastTimestamp: "2026-07-13T19:01:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:55Z" name: error-404-isvc-19366-predictor.18c1ee9ce5bedfd4 namespace: kserve-ci-e2e-test resourceVersion: "15555" uid: 1a27ca1a-ca07-4982-bcab-354c217dfad3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19366 namespace: kserve-ci-e2e-test resourceVersion: "14598" uid: 4256b5bb-df5d-42ae-af9e-bad7e1116992 kind: Event lastTimestamp: "2026-07-13T19:01:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:31Z" name: error-404-isvc-19366.18c1ee8eea3b7f98 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: e7524763-df9a-4ef1-adea-10ae89cabda6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-19366 namespace: kserve-ci-e2e-test resourceVersion: "15323" uid: 4256b5bb-df5d-42ae-af9e-bad7e1116992 kind: Event lastTimestamp: "2026-07-13T19:01:31Z" message: InferenceService [error-404-isvc-19366] is Ready metadata: creationTimestamp: "2026-07-13T19:01:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:31Z" name: error-404-isvc-19366.18c1ee9e79e2e0a0 namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 695575da-d179-4ee9-acf1-d008fdb79209 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:44.778494Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3a16a-predictor-59bd97769f-hwr4r to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:44Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf75cd4a1f namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: e9790ca6-2ca1-43fb-84bf-ece34e70fa0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16290" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf8fbab374 namespace: kserve-ci-e2e-test resourceVersion: "16309" uid: ca9547ba-80f3-438b-9c9d-5d3fe738ab8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf91cc847e namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 647c783d-466e-444c-ae82-c2ca3a8b61d4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf9442ee1b namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: b29731bd-9e9b-4864-929a-c0441f1f4e17 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf94b43851 namespace: kserve-ci-e2e-test resourceVersion: "16318" uid: cfa7278b-5f4f-4110-a904-d79261fef5bf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf94bf3ca0 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 81811ac6-483f-4abc-9c51-1f19423a5171 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf975e1244 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: 80f23b2c-4e7b-49d9-b688-d169d679a7c7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeaf97d0b650 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 696c45c0-6606-4517-b078-fcde46fe43cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:12:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:02Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1eeafcb4c18d3 namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: 879d8474-f5ed-4777-8f74-5db4ad649a06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1ef30a848e134 namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: dc34744e-cdf8-40eb-a584-8923dcd5c74e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1ef30a849a6c3 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: a5b572cd-efa5-456e-8054-60f72d14f775 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: cb069095-a5f5-45bd-9451-6f3b145a1219 kind: Event lastTimestamp: "2026-07-13T19:12:02Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:02Z" name: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r.18c1ef313fc3a30e namespace: kserve-ci-e2e-test resourceVersion: "20631" uid: b0a80dbb-f2a5-48c9-a1e1-04b9f8046fb6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3a16a-predictor-59bd97769f namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: c0ba881e-55ad-443f-b161-87e6e941672d kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: 'Created pod: error-404-isvc-3a16a-predictor-59bd97769f-hwr4r' metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:44Z" name: error-404-isvc-3a16a-predictor-59bd97769f.18c1eeaf757004ea namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: a15d027b-11f5-46ff-968a-d290255e94db reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16255" uid: 8220c72d-1020-4bbe-b102-90bdea33c3d2 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Scaled up replica set error-404-isvc-3a16a-predictor-59bd97769f from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:44Z" name: error-404-isvc-3a16a-predictor.18c1eeaf73978ec0 namespace: kserve-ci-e2e-test resourceVersion: "16259" uid: 9087a445-24f9-4878-a178-acb88f255543 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 52e47848-69b2-4128-ba2a-0db8c88e1ecf kind: Event lastTimestamp: "2026-07-13T19:03:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:14Z" name: error-404-isvc-3a16a-predictor.18c1eeb2f25b298a namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: fb422910-b334-478a-83e6-9c07e3b39f8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16256" uid: 52e47848-69b2-4128-ba2a-0db8c88e1ecf kind: Event lastTimestamp: "2026-07-13T19:03:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:14Z" name: error-404-isvc-3a16a-predictor.18c1eeb2f2df88b5 namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: ea8579f1-6090-4f25-8bb7-a24974de0091 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 52e47848-69b2-4128-ba2a-0db8c88e1ecf kind: Event lastTimestamp: "2026-07-13T19:03:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:59Z" name: error-404-isvc-3a16a-predictor.18c1eeb9f063ad3a namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: e007e81c-2a2d-4199-95e2-9d2d0a7caba6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16477" uid: 52e47848-69b2-4128-ba2a-0db8c88e1ecf kind: Event lastTimestamp: "2026-07-13T19:03:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:59Z" name: error-404-isvc-3a16a-predictor.18c1eeb9f0d8688d namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: e8a97c20-e3d3-471c-a034-9f43e6da5afa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 5fce4e22-d4eb-428d-934a-b1ffaa562d2c kind: Event lastTimestamp: "2026-07-13T19:03:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:32Z" name: error-404-isvc-3a16a.18c1eeaf739ec51f namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 7feda02a-ee0a-4be4-9031-bc3d79783e59 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 5fce4e22-d4eb-428d-934a-b1ffaa562d2c kind: Event lastTimestamp: "2026-07-13T19:03:32Z" message: InferenceService [error-404-isvc-3a16a] is Ready metadata: creationTimestamp: "2026-07-13T19:03:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:32Z" name: error-404-isvc-3a16a.18c1eeba8afa1951 namespace: kserve-ci-e2e-test resourceVersion: "16785" uid: f5eb2ccf-3d11-41d0-920e-3173304c09da reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:22:44.845286Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc6df5b583d namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: af45a69b-5938-4b7a-8fec-854e47d38350 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4dc86-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc6ea7ca390 namespace: kserve-ci-e2e-test resourceVersion: "26284" uid: 4a59fdbf-3c6e-4242-b105-a3b004dce02f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc71da039c6 namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: cb735b09-d3f5-4ed8-a7b7-dc18974d30cf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc71f2aa40d namespace: kserve-ci-e2e-test resourceVersion: "26305" uid: f29a7aec-4301-488e-8617-ca4e69b2c027 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc721c0f27b namespace: kserve-ci-e2e-test resourceVersion: "26308" uid: b7fecb6c-14e4-4746-8be5-c6c4b71eb6da reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc7223e9b33 namespace: kserve-ci-e2e-test resourceVersion: "26309" uid: e5291f78-d89e-4cc4-abdf-0bb632edde3d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc722494090 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 13cd9351-e815-4efb-96f1-0683631da707 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:46Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc7258d75bf namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: 357d9a01-990f-4b6c-a751-08658b09b3f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:22:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:46Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc726042360 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: 57b52242-1fa9-454b-8d20-78792aef395d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:23:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:23Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1efc75fb07004 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 103305f5-1146-497d-8ca2-26d6be64c7d1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:40:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:14Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1f0bb3ef3d78e namespace: kserve-ci-e2e-test resourceVersion: "34061" uid: 7a8c21f6-991e-487d-814d-a6ee56437974 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr namespace: kserve-ci-e2e-test resourceVersion: "26268" uid: 5cf5f081-81e8-4cd2-a14e-4d235593ae37 kind: Event lastTimestamp: "2026-07-13T19:40:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:14Z" name: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr.18c1f0bb3ef44d33 namespace: kserve-ci-e2e-test resourceVersion: "34062" uid: 16e746a8-499b-4e17-9d9a-c871cd9fabec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4dc86-predictor-79bccf84c7 namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: e0f300bd-780a-4d71-9896-1c5bdce59a48 kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: 'Created pod: error-404-isvc-4dc86-predictor-79bccf84c7-zpzcr' metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-4dc86-predictor-79bccf84c7.18c1efc6def14d25 namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: a879c59b-1c64-4d28-b77f-b0637a165c82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: a3da29d2-54bb-4833-a19a-d81470f224a0 kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Scaled up replica set error-404-isvc-4dc86-predictor-79bccf84c7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-4dc86-predictor.18c1efc6dd14a35f namespace: kserve-ci-e2e-test resourceVersion: "26260" uid: 75c55443-998a-4e89-846c-b74e32b1e280 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 05892ac6-4af1-436f-b71f-3cd3e0df091f kind: Event lastTimestamp: "2026-07-13T19:23:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:14Z" name: error-404-isvc-4dc86-predictor.18c1efca5bb72c9b namespace: kserve-ci-e2e-test resourceVersion: "26585" uid: fb41b8c6-d5f8-449c-a45e-4c6a295bc353 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 05892ac6-4af1-436f-b71f-3cd3e0df091f kind: Event lastTimestamp: "2026-07-13T19:23:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:14Z" name: error-404-isvc-4dc86-predictor.18c1efca5c0bdd65 namespace: kserve-ci-e2e-test resourceVersion: "26586" uid: 496f7621-ebf0-45a6-8970-33be455e3f91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 05892ac6-4af1-436f-b71f-3cd3e0df091f kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: error-404-isvc-4dc86-predictor.18c1efd159a1cd52 namespace: kserve-ci-e2e-test resourceVersion: "27186" uid: 57bde39e-6e26-4f8f-8914-a0cce799fc03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26450" uid: 05892ac6-4af1-436f-b71f-3cd3e0df091f kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: error-404-isvc-4dc86-predictor.18c1efd15a0e2658 namespace: kserve-ci-e2e-test resourceVersion: "27187" uid: 261ad69e-6dac-4ad5-ac90-96bcf05d681d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 5d3c0fe2-0411-449b-8068-7a8009cb0b7c kind: Event lastTimestamp: "2026-07-13T19:23:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:33Z" name: error-404-isvc-4dc86.18c1efc6dd1b0956 namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 264b5666-b15c-4d54-9cff-6b813d4e90d6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 5d3c0fe2-0411-449b-8068-7a8009cb0b7c kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: 'Failed to update status for InferenceService "error-404-isvc-4dc86": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4dc86": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-4dc86.18c1efc6ddab572e namespace: kserve-ci-e2e-test resourceVersion: "26263" uid: 202ebb12-4004-4ef9-b4a6-7dc78904fb76 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 5d3c0fe2-0411-449b-8068-7a8009cb0b7c kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4dc86": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-4dc86.18c1efc6ddab97cb namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: 2522423d-d7c3-4f02-8379-aa487659567b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: 5d3c0fe2-0411-449b-8068-7a8009cb0b7c kind: Event lastTimestamp: "2026-07-13T19:23:33Z" message: InferenceService [error-404-isvc-4dc86] is Ready metadata: creationTimestamp: "2026-07-13T19:23:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:33Z" name: error-404-isvc-4dc86.18c1efd21d51ffb1 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 678cef62-b7d2-4f40-b177-a8c109583f5e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:23.751207Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20127" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-68f71-predictor-6478fb4ccf-w9blh to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef284afc78b6 namespace: kserve-ci-e2e-test resourceVersion: "20131" uid: 7bb8305e-781d-4ef5-a813-a5340142ea9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef2864a2c075 namespace: kserve-ci-e2e-test resourceVersion: "20167" uid: 8b179d5f-cad8-498a-b76f-47ced4d3e994 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef286665be31 namespace: kserve-ci-e2e-test resourceVersion: "20169" uid: 6ba0be43-e0ce-46be-8d13-f6391b350b8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef286921dd2c namespace: kserve-ci-e2e-test resourceVersion: "20170" uid: cd4d17a3-8247-474f-8a29-c1b42e500de6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef28699218c9 namespace: kserve-ci-e2e-test resourceVersion: "20171" uid: 2a51d786-1fc9-4625-9f93-6c50f8cd5ed3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef28699b25f4 namespace: kserve-ci-e2e-test resourceVersion: "20172" uid: 28feeea0-fa01-492b-a057-7cab53ce6c81 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef286c3f71cc namespace: kserve-ci-e2e-test resourceVersion: "20174" uid: 4b7c672c-a2e4-42e3-a769-bdeaaad3dccf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef286cbb7f67 namespace: kserve-ci-e2e-test resourceVersion: "20175" uid: 04d48e26-4864-4007-b754-aa517e4385bd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:11:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:12:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:11:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:11Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef28a38b1aa9 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: 69472220-4ca8-4c8b-bf48-1a786f898fbe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:43Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef3af408a1d3 namespace: kserve-ci-e2e-test resourceVersion: "21088" uid: 703cea6f-dbb9-4b2f-9ef7-10a3abeb151a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:43Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef3af4091c70 namespace: kserve-ci-e2e-test resourceVersion: "21091" uid: a318e616-798f-4e21-8084-a2c51294fcc5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh namespace: kserve-ci-e2e-test resourceVersion: "20130" uid: 7377b88d-c036-4ff7-89de-fab1bc293e1a kind: Event lastTimestamp: "2026-07-13T19:12:46Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:46Z" name: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh.18c1ef3b7fc9ad8a namespace: kserve-ci-e2e-test resourceVersion: "21191" uid: 74cb8b74-db8d-49ed-9982-52347f1302f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-68f71-predictor-6478fb4ccf namespace: kserve-ci-e2e-test resourceVersion: "20118" uid: b51d601a-710e-4d05-941c-47aa7bdd51c4 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'Created pod: error-404-isvc-68f71-predictor-6478fb4ccf-w9blh' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-68f71-predictor-6478fb4ccf.18c1ef284a555247 namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: 58041a52-8917-46d8-8c1d-d944dbb4340f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: e903c847-410e-429c-97e1-4bc51beb75e2 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Scaled up replica set error-404-isvc-68f71-predictor-6478fb4ccf from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-68f71-predictor.18c1ef28482ed940 namespace: kserve-ci-e2e-test resourceVersion: "20120" uid: 8cf3d1d9-e994-429b-8469-f73f1217a020 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:11:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: d708d71e-51d3-4e33-b764-4fb0c1264e89 kind: Event lastTimestamp: "2026-07-13T19:12:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:38Z" name: error-404-isvc-68f71-predictor.18c1ef2bc77c26d9 namespace: kserve-ci-e2e-test resourceVersion: "20989" uid: 1f533f1a-da17-4aae-a2cf-bc31c18c82eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:11:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: d708d71e-51d3-4e33-b764-4fb0c1264e89 kind: Event lastTimestamp: "2026-07-13T19:12:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:38Z" name: error-404-isvc-68f71-predictor.18c1ef2bc900a1e9 namespace: kserve-ci-e2e-test resourceVersion: "20990" uid: b8f04e11-f69e-44ad-acd0-af5480dd050a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: d708d71e-51d3-4e33-b764-4fb0c1264e89 kind: Event lastTimestamp: "2026-07-13T19:11:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:53Z" name: error-404-isvc-68f71-predictor.18c1ef2f481804f6 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 9c09334a-25aa-4ada-98bf-4544bc48620f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20313" uid: d708d71e-51d3-4e33-b764-4fb0c1264e89 kind: Event lastTimestamp: "2026-07-13T19:11:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:53Z" name: error-404-isvc-68f71-predictor.18c1ef2f4871782c namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: d96d82bf-24fe-4a33-847a-81b642c11929 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acf50438-ef51-4259-a444-1d812ee45ff6 kind: Event lastTimestamp: "2026-07-13T19:12:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:21Z" name: error-404-isvc-68f71.18c1ef28487fdbb1 namespace: kserve-ci-e2e-test resourceVersion: "20802" uid: fe96eec1-48ff-4cc1-8e36-4b7ce768efb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acf50438-ef51-4259-a444-1d812ee45ff6 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'Failed to update status for InferenceService "error-404-isvc-68f71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68f71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-68f71.18c1ef28491093b8 namespace: kserve-ci-e2e-test resourceVersion: "20124" uid: 865fef10-b45b-42f2-8e37-57ce81b4fdec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: acf50438-ef51-4259-a444-1d812ee45ff6 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-68f71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-68f71.18c1ef284910d301 namespace: kserve-ci-e2e-test resourceVersion: "20126" uid: 6e515388-6c4d-4e91-baea-f1831b79965d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: acf50438-ef51-4259-a444-1d812ee45ff6 kind: Event lastTimestamp: "2026-07-13T19:12:21Z" message: InferenceService [error-404-isvc-68f71] is Ready metadata: creationTimestamp: "2026-07-13T19:12:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:21Z" name: error-404-isvc-68f71.18c1ef35b31834f5 namespace: kserve-ci-e2e-test resourceVersion: "20801" uid: a59d597d-f8e2-45ab-b37a-bb7ecc988fc9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:01:58.803590Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15684" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-91c3d-predictor-84d6544c-dqr8h to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea4c17bfef8 namespace: kserve-ci-e2e-test resourceVersion: "15690" uid: 31725d63-6544-4626-ba0f-b143d595b611 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-91c3d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea4ccda0c33 namespace: kserve-ci-e2e-test resourceVersion: "15703" uid: 5f1119cd-926c-4ed5-8fd2-30a644f63176 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15696" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Add eth0 [10.134.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:01:59Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea4fea23493 namespace: kserve-ci-e2e-test resourceVersion: "15725" uid: d392907a-78df-43cf-ba70-d3b8f3e2e6ce reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea502180633 namespace: kserve-ci-e2e-test resourceVersion: "15727" uid: c472758e-9776-4d28-9bb7-f21f81d9a83a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea505756de9 namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: bbdc8fc8-8517-4148-9b74-3f0ace735016 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea505e7430e namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: de578225-5744-4d99-aaa8-e32b4b3ede74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea505f21d95 namespace: kserve-ci-e2e-test resourceVersion: "15731" uid: 870b78cc-11e1-402a-9bee-6d740955b701 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:02:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:00Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea5096e8f46 namespace: kserve-ci-e2e-test resourceVersion: "15732" uid: b9775cee-de58-45ee-ad7e-3b98ca3709f4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:02:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:00Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea509f456ed namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 4643d0b6-3b80-4c7c-acd0-c87255813041 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:02:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:11:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:27Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1eea5460e53da namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 4894ad98-3b43-4d01-9a62-d76b482a5a17 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1ef284fd1f025 namespace: kserve-ci-e2e-test resourceVersion: "20154" uid: 6b7012a5-f9a4-4f9c-8462-a37f1f41015b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1ef284fd26440 namespace: kserve-ci-e2e-test resourceVersion: "20156" uid: adf7afb7-2c0a-4245-8e55-3d86d00c5e89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h namespace: kserve-ci-e2e-test resourceVersion: "15687" uid: 5b77ffb2-b4b5-462f-ba67-092843462fb4 kind: Event lastTimestamp: "2026-07-13T19:11:27Z" message: 'Readiness probe failed: Get "https://10.134.0.25:8643/healthz": dial tcp 10.134.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:27Z" name: error-404-isvc-91c3d-predictor-84d6544c-dqr8h.18c1ef290ea4159b namespace: kserve-ci-e2e-test resourceVersion: "20210" uid: 67778e71-8007-4669-a254-5b801ff133b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-91c3d-predictor-84d6544c namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: 870f9de1-45ac-4b00-af83-d878b39dea09 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'Created pod: error-404-isvc-91c3d-predictor-84d6544c-dqr8h' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d-predictor-84d6544c.18c1eea4c10b749d namespace: kserve-ci-e2e-test resourceVersion: "15686" uid: 3b26b0e6-32f1-4e24-8a47-00eafa9f0855 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15673" uid: bbfcf686-6ddb-4f10-a176-55aa58356ac1 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Scaled up replica set error-404-isvc-91c3d-predictor-84d6544c from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d-predictor.18c1eea4bf4d21b3 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 39d62c2e-802d-4d89-a94c-9691f77dd57a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 2da28a4f-11c4-4332-8b55-75fb5599e186 kind: Event lastTimestamp: "2026-07-13T19:02:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:28Z" name: error-404-isvc-91c3d-predictor.18c1eea83dd31d49 namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: 0eb6b20e-2b2d-4514-85ff-fba1750c0d7a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: 2da28a4f-11c4-4332-8b55-75fb5599e186 kind: Event lastTimestamp: "2026-07-13T19:02:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:28Z" name: error-404-isvc-91c3d-predictor.18c1eea83e33ac26 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: 26f63a04-6c6b-4ed4-b704-780d958db9a3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 2da28a4f-11c4-4332-8b55-75fb5599e186 kind: Event lastTimestamp: "2026-07-13T19:03:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:13Z" name: error-404-isvc-91c3d-predictor.18c1eeaf3bac9fc1 namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 1efe0d08-8841-41f3-84c9-66e82780e991 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15876" uid: 2da28a4f-11c4-4332-8b55-75fb5599e186 kind: Event lastTimestamp: "2026-07-13T19:03:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:13Z" name: error-404-isvc-91c3d-predictor.18c1eeaf3c17510f namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: c61da617-db57-4613-bc7e-1922e213cf70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: ed183c53-f381-400f-a8cd-4b02ecf512d4 kind: Event lastTimestamp: "2026-07-13T19:02:47Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:47Z" name: error-404-isvc-91c3d.18c1eea4bf51db65 namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: 05f9a409-d6ef-469c-ad60-2f1283727344 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: ed183c53-f381-400f-a8cd-4b02ecf512d4 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'Failed to update status for InferenceService "error-404-isvc-91c3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d.18c1eea4bfd613bb namespace: kserve-ci-e2e-test resourceVersion: "15680" uid: cdc3ce96-4380-43c3-ab90-a87f1c5d5f15 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15642" uid: ed183c53-f381-400f-a8cd-4b02ecf512d4 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-91c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:58Z" name: error-404-isvc-91c3d.18c1eea4bfd661fb namespace: kserve-ci-e2e-test resourceVersion: "15681" uid: 4140021a-7df8-4768-800c-52b1723bbf9c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: ed183c53-f381-400f-a8cd-4b02ecf512d4 kind: Event lastTimestamp: "2026-07-13T19:02:47Z" message: InferenceService [error-404-isvc-91c3d] is Ready metadata: creationTimestamp: "2026-07-13T19:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:47Z" name: error-404-isvc-91c3d.18c1eeb00757e5bc namespace: kserve-ci-e2e-test resourceVersion: "16352" uid: 211d8e0a-7075-4063-98ab-f40698282f95 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:13:20.053298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a68dc-predictor-8656958995-dh6rm to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef435f218487 namespace: kserve-ci-e2e-test resourceVersion: "21624" uid: 086c1dac-a6e0-403d-8091-01df92b65802 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21630" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef4379306070 namespace: kserve-ci-e2e-test resourceVersion: "21647" uid: 857409a0-eddc-4bfc-9a70-b73dbc679c8e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef437aaaf4b2 namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: d275579c-2dab-4b4c-b820-9767773d321d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef437cea4be2 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: c28db36e-789e-47d0-bda3-581b0aec6eca reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef437d5fdaaa namespace: kserve-ci-e2e-test resourceVersion: "21657" uid: 1fab9738-0963-4dd4-be4a-9536a2c1de43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef437d69e541 namespace: kserve-ci-e2e-test resourceVersion: "21658" uid: cdd93429-b239-430e-bde6-2c6f190597dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef4380a16d36 namespace: kserve-ci-e2e-test resourceVersion: "21659" uid: 51965d49-0b83-4f91-8006-7e327f6fa6a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef4381374e58 namespace: kserve-ci-e2e-test resourceVersion: "21660" uid: b16c7f45-7702-4661-991a-e8faef424756 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:22:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:47Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1ef43c552359b namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: bfa4d793-28a1-4a4c-9930-5a71964d61cd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1efc6d84c2cbf namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 51149cfd-9387-49d4-91c1-c2ae9e8b8105 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1efc6d84cb9dc namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 5b7b5e87-639c-412b-ae55-0535ac651a4b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a68dc-predictor-8656958995-dh6rm namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: a9e26c55-973f-41d1-bfb5-c1af71938dff kind: Event lastTimestamp: "2026-07-13T19:22:47Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:47Z" name: error-404-isvc-a68dc-predictor-8656958995-dh6rm.18c1efc78dbeb9e9 namespace: kserve-ci-e2e-test resourceVersion: "26340" uid: ca3f321a-cfc0-4110-a2e1-19d80911afdd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a68dc-predictor-8656958995 namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: cdbe7abc-179a-4d6b-8496-03724609f32e kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Created pod: error-404-isvc-a68dc-predictor-8656958995-dh6rm' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor-8656958995.18c1ef435e8be876 namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 9fac4d35-855f-46c5-80b2-d30cb8ea8c77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: d672e1d9-4d3c-47c1-baab-9f3635622fcc kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Scaled up replica set error-404-isvc-a68dc-predictor-8656958995 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc-predictor.18c1ef435cf1a659 namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: 0a94e848-8bdd-4943-a84e-ae37e9c5ced6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:13:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: 0a85f32f-b866-4bc1-a9f7-f8cefffa0478 kind: Event lastTimestamp: "2026-07-13T19:14:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:05Z" name: error-404-isvc-a68dc-predictor.18c1ef46db74396d namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 8affda34-3037-4c02-9a1a-37cd97c57e9c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:13:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21612" uid: 0a85f32f-b866-4bc1-a9f7-f8cefffa0478 kind: Event lastTimestamp: "2026-07-13T19:14:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:05Z" name: error-404-isvc-a68dc-predictor.18c1ef46dbf78d67 namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: 067ca8d1-f4de-44dd-9b39-2d27ef54c1cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:14:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 0a85f32f-b866-4bc1-a9f7-f8cefffa0478 kind: Event lastTimestamp: "2026-07-13T19:14:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:35Z" name: error-404-isvc-a68dc-predictor.18c1ef5158195f35 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: deed12e4-f79b-4e19-aa0e-11b9344bce3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:14:20Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21793" uid: 0a85f32f-b866-4bc1-a9f7-f8cefffa0478 kind: Event lastTimestamp: "2026-07-13T19:14:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:14:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:35Z" name: error-404-isvc-a68dc-predictor.18c1ef5158881deb namespace: kserve-ci-e2e-test resourceVersion: "22361" uid: c9cb201a-8ab5-4843-88f7-9beadef258c2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 52caec64-be76-421a-a1ea-077e102af82a kind: Event lastTimestamp: "2026-07-13T19:14:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:07Z" name: error-404-isvc-a68dc.18c1ef435cfbd449 namespace: kserve-ci-e2e-test resourceVersion: "22115" uid: 36e5093d-36d3-435a-a0e6-39a965e309dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 52caec64-be76-421a-a1ea-077e102af82a kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-a68dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a68dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc.18c1ef435d71f470 namespace: kserve-ci-e2e-test resourceVersion: "21617" uid: dfe51543-8615-4ed6-939d-837d9f0aa254 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21599" uid: 52caec64-be76-421a-a1ea-077e102af82a kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a68dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:20Z" name: error-404-isvc-a68dc.18c1ef435d7232b7 namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: c0e09d85-0215-4bdb-9253-0aafaf16d393 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 52caec64-be76-421a-a1ea-077e102af82a kind: Event lastTimestamp: "2026-07-13T19:14:07Z" message: InferenceService [error-404-isvc-a68dc] is Ready metadata: creationTimestamp: "2026-07-13T19:14:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:07Z" name: error-404-isvc-a68dc.18c1ef4e8184d533 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: f39a1b10-18f2-447e-afc5-ec81c8fc654e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:22:08.991300Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25827" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-b5afb-predictor-6754b9b7-xxv6p to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:22:08Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbe864af4a4 namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: 1a19ba36-0a22-4e2b-8f86-38ace35a01e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-b5afb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:09Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbe94bdcfac namespace: kserve-ci-e2e-test resourceVersion: "25861" uid: b1b2cbca-dba2-4b33-92fe-77f3bfb6dfbc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbec5131b8d namespace: kserve-ci-e2e-test resourceVersion: "25875" uid: 2f98e27a-3bb5-47e5-a7d0-9a7c2e9ef28c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbec680c892 namespace: kserve-ci-e2e-test resourceVersion: "25880" uid: 3f34a7d0-5d49-4a4a-8d80-44c8c2cd081d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbec9e75408 namespace: kserve-ci-e2e-test resourceVersion: "25883" uid: 1235fc25-a577-47e6-b30e-4a5af2804b5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbeca65f0b6 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 3c64a66c-445e-40da-888a-b51ed2122a43 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbeca709703 namespace: kserve-ci-e2e-test resourceVersion: "25886" uid: 1a25bcc6-6158-4435-becd-a53d4d130b5c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbecd0f2aaf namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: b386a932-77c1-4b6e-8028-279f7692cc06 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbecd80e7db namespace: kserve-ci-e2e-test resourceVersion: "25888" uid: 0ea5f6d9-3c89-41cb-8f1b-cbb7d4058723 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:22:56Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:56Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efbef4760d22 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: 9a179334-508d-4a52-9dd3-d2ed22f13ce0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efd1339eb610 namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: 07692c5d-9b17-4b92-8c6f-8f2020cd4df4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efd1339f2caa namespace: kserve-ci-e2e-test resourceVersion: "26794" uid: 369dc797-3d3f-40f9-866a-e3136dca62a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 6194efd1-4acd-483c-9c77-2dc1b5630ebe kind: Event lastTimestamp: "2026-07-13T19:23:31Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:31Z" name: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p.18c1efd1d09604c9 namespace: kserve-ci-e2e-test resourceVersion: "26893" uid: 09231a1c-6e25-4b97-9e67-728a0e5282c8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-b5afb-predictor-6754b9b7 namespace: kserve-ci-e2e-test resourceVersion: "25819" uid: 14f384b8-0419-42e5-985f-65289f3a7dfe kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'Created pod: error-404-isvc-b5afb-predictor-6754b9b7-xxv6p' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:08Z" name: error-404-isvc-b5afb-predictor-6754b9b7.18c1efbe85d0357c namespace: kserve-ci-e2e-test resourceVersion: "25828" uid: a8481a37-7cf7-480a-a887-22d9581d9f0b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: 73e80026-a14a-421c-9216-9b38bc3e9936 kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: Scaled up replica set error-404-isvc-b5afb-predictor-6754b9b7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:08Z" name: error-404-isvc-b5afb-predictor.18c1efbe839ba6b5 namespace: kserve-ci-e2e-test resourceVersion: "25821" uid: ee0b1cb5-7ad9-490c-a71c-084e6256b72a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:22:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 3e9c468f-ba95-4208-a8eb-13f71a321a8b kind: Event lastTimestamp: "2026-07-13T19:23:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:24Z" name: error-404-isvc-b5afb-predictor.18c1efc2023594ed namespace: kserve-ci-e2e-test resourceVersion: "26690" uid: 0c1573b1-2ce3-4061-b18a-e8c5ac9197a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:22:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25817" uid: 3e9c468f-ba95-4208-a8eb-13f71a321a8b kind: Event lastTimestamp: "2026-07-13T19:23:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:24Z" name: error-404-isvc-b5afb-predictor.18c1efc2028b7d0d namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: a7b92ca4-11b5-4ae2-a2ea-4d848269059f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 7715ef36-07ce-4670-89a3-52cc06319adb kind: Event lastTimestamp: "2026-07-13T19:23:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:06Z" name: error-404-isvc-b5afb.18c1efbe83bed032 namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: 914c5e4c-3891-4bb5-8287-04c5e5343887 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 7715ef36-07ce-4670-89a3-52cc06319adb kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-b5afb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b5afb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:08Z" name: error-404-isvc-b5afb.18c1efbe8445b809 namespace: kserve-ci-e2e-test resourceVersion: "25824" uid: 120d4fa1-2f88-4c4b-9d69-64a42318f225 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 7715ef36-07ce-4670-89a3-52cc06319adb kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-b5afb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:08Z" name: error-404-isvc-b5afb.18c1efbe844608c3 namespace: kserve-ci-e2e-test resourceVersion: "25826" uid: a14d2811-0d5f-4e96-be96-436a7beed184 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "26505" uid: 7715ef36-07ce-4670-89a3-52cc06319adb kind: Event lastTimestamp: "2026-07-13T19:23:06Z" message: InferenceService [error-404-isvc-b5afb] is Ready metadata: creationTimestamp: "2026-07-13T19:23:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:06Z" name: error-404-isvc-b5afb.18c1efcc043d3ba6 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: 8f2b48ce-ab07-49cc-8eed-7fedd0dc80cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:59.901827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20565" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30b5bb19cd namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 5b2b680b-dc2d-4304-8907-8224997c0f5f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20579" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30cfa8dbb1 namespace: kserve-ci-e2e-test resourceVersion: "20592" uid: 22edba16-02e1-4d72-a1fa-17671a4a7760 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d1521257 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: 2b6f4298-34eb-4e63-b4da-3d25cb786703 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d44d92fc namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 926f0080-9501-43e0-9132-1723d44883cd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d4c73949 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: cf0016c8-2354-4ae6-a734-c6b7d84fab05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d4d23779 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 8d6409e5-3a68-42d8-bf81-f8591ebb3b63 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d7eb0176 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 6bbe9099-ff44-4750-a5c1-75b869e7aad5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef30d86199d0 namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: dd8c8cfd-31fc-4f59-b6ad-ca3be327a3ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:12:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:47Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef310eae18bb namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 797195f2-6cfd-48fa-b20f-2dc61f49df4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:19Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef4357cc8087 namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 5604ce4e-0048-4ef6-bef8-4a627df9fab6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:19Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef4357ccffd1 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: c11d949b-5c58-4296-97cc-0d98c0d7610b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8 namespace: kserve-ci-e2e-test resourceVersion: "20567" uid: 91e5a392-9660-43eb-8494-ee0c8eb2702e kind: Event lastTimestamp: "2026-07-13T19:13:22Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:22Z" name: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8.18c1ef43eb05ac44 namespace: kserve-ci-e2e-test resourceVersion: "21684" uid: 426bd29d-0d13-49b3-b041-6232554f43cc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e56dd-predictor-7f8878c579 namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 18cba489-96d2-4d2b-bf15-b906310122e0 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'Created pod: error-404-isvc-e56dd-predictor-7f8878c579-l8lf8' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-e56dd-predictor-7f8878c579.18c1ef30b5131f16 namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: da0c337e-f649-40eb-8ee4-1b055cf4f99a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20556" uid: 12603854-900e-4a5c-b9f3-821d5ebedf67 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Scaled up replica set error-404-isvc-e56dd-predictor-7f8878c579 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-e56dd-predictor.18c1ef30b24cfe9f namespace: kserve-ci-e2e-test resourceVersion: "20559" uid: dce4ccd3-c47b-4c14-b1a8-89f58f6d2942 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 3807c136-674b-4ad4-a224-08f12e53d30e kind: Event lastTimestamp: "2026-07-13T19:12:29Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:29Z" name: error-404-isvc-e56dd-predictor.18c1ef3430bbf00f namespace: kserve-ci-e2e-test resourceVersion: "20886" uid: 6ff86d30-3e64-41df-92ff-eb6184910e36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 3807c136-674b-4ad4-a224-08f12e53d30e kind: Event lastTimestamp: "2026-07-13T19:12:29Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:29Z" name: error-404-isvc-e56dd-predictor.18c1ef3431295284 namespace: kserve-ci-e2e-test resourceVersion: "20887" uid: 2071cabe-b657-4c1f-973e-35ad15bbe13f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 3807c136-674b-4ad4-a224-08f12e53d30e kind: Event lastTimestamp: "2026-07-13T19:13:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:14Z" name: error-404-isvc-e56dd-predictor.18c1ef3b2eb5d6ae namespace: kserve-ci-e2e-test resourceVersion: "21481" uid: 955d152a-c9f0-4885-9260-255b20eb6149 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20748" uid: 3807c136-674b-4ad4-a224-08f12e53d30e kind: Event lastTimestamp: "2026-07-13T19:13:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:14Z" name: error-404-isvc-e56dd-predictor.18c1ef3b2f22a480 namespace: kserve-ci-e2e-test resourceVersion: "21482" uid: ed42a9b4-1854-476d-872a-278f92c5d34a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: ef17d729-2eea-4fa6-9365-c6e2aa933d7c kind: Event lastTimestamp: "2026-07-13T19:12:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:57Z" name: error-404-isvc-e56dd.18c1ef30b25049f4 namespace: kserve-ci-e2e-test resourceVersion: "21307" uid: dfec19dc-92a4-4d5b-be3a-187df72570cc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: ef17d729-2eea-4fa6-9365-c6e2aa933d7c kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'Failed to update status for InferenceService "error-404-isvc-e56dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e56dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-e56dd.18c1ef30b352acb1 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: b305b65a-13ce-4f8e-985a-b28b7370bf46 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: ef17d729-2eea-4fa6-9365-c6e2aa933d7c kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e56dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:59Z" name: error-404-isvc-e56dd.18c1ef30b353022c namespace: kserve-ci-e2e-test resourceVersion: "20563" uid: 971c47ad-5c58-4bf1-8fa1-4e07964116af reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "21303" uid: ef17d729-2eea-4fa6-9365-c6e2aa933d7c kind: Event lastTimestamp: "2026-07-13T19:12:57Z" message: InferenceService [error-404-isvc-e56dd] is Ready metadata: creationTimestamp: "2026-07-13T19:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:57Z" name: error-404-isvc-e56dd.18c1ef3e21061981 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: eb11d657-2d9b-4391-9552-bceabefbb5cb reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:12:44.024235Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21124" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ef1db-predictor-867dc684b7-94jfl to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:12:44Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3afba19132 namespace: kserve-ci-e2e-test resourceVersion: "21128" uid: 5d661ce7-c219-438d-81a7-25e31554d59c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ef1db-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:44Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b037215a2 namespace: kserve-ci-e2e-test resourceVersion: "21143" uid: fc28bcf5-acc8-459d-8cf6-883eb8ca6942 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21134" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b3a49e7fe namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: 151f6069-4c95-45b7-9a9f-8198647e1122 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b3bd61fd7 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 88c2551f-7b5f-4909-97b3-a1884d9d4e68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b3f5de92b namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: d648b97b-d5fb-4144-921d-4fbb0f9de1c8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b3fd4ff82 namespace: kserve-ci-e2e-test resourceVersion: "21171" uid: 8ae543dd-b6e0-4c1c-a268-e7a6149fbe0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b3fdfad09 namespace: kserve-ci-e2e-test resourceVersion: "21172" uid: 85aac0ef-24c6-4f53-911b-90eecd20d633 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b43f5a52f namespace: kserve-ci-e2e-test resourceVersion: "21173" uid: 4ffb99d2-ac26-455a-a7a5-e14a82263aaa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b4473ef4e namespace: kserve-ci-e2e-test resourceVersion: "21174" uid: c77ca25e-654b-4f76-b0a5-edd597f09174 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:22:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:11Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1ef3b593f5cdf namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: b167e843-38aa-4e3b-bbc8-e6736fa0b303 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:09Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1efbe8a07843e namespace: kserve-ci-e2e-test resourceVersion: "25850" uid: 2564a353-55fa-4c0f-9af6-8d942a21587c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:09Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1efbe8a081177 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: b5085117-a956-4af0-824a-e760e3c88cb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 7e30082a-3681-4bbc-8ad0-6e5067149ce5 kind: Event lastTimestamp: "2026-07-13T19:22:11Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:11Z" name: error-404-isvc-ef1db-predictor-867dc684b7-94jfl.18c1efbf21bf5f64 namespace: kserve-ci-e2e-test resourceVersion: "25906" uid: d9345688-13a9-495c-94c9-2919d4e16632 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ef1db-predictor-867dc684b7 namespace: kserve-ci-e2e-test resourceVersion: "21117" uid: da36af3a-afc4-44cd-bd66-265578d24fe1 kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: 'Created pod: error-404-isvc-ef1db-predictor-867dc684b7-94jfl' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:44Z" name: error-404-isvc-ef1db-predictor-867dc684b7.18c1ef3afb1d3025 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: 5788e202-a4bf-4ce7-8a7e-f5a3188ece0f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 4abfdcfb-a442-4deb-a605-350a75215816 kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Scaled up replica set error-404-isvc-ef1db-predictor-867dc684b7 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:43Z" name: error-404-isvc-ef1db-predictor.18c1ef3af99f141a namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 6ead5858-25ae-493d-ae48-78f67630f2fb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 8e898d56-d325-429e-892b-1ea8143582cc kind: Event lastTimestamp: "2026-07-13T19:12:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:58Z" name: error-404-isvc-ef1db-predictor.18c1ef3e7822777f namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 73e0017e-342c-4195-a739-d1695a38c475 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 8e898d56-d325-429e-892b-1ea8143582cc kind: Event lastTimestamp: "2026-07-13T19:12:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:59Z" name: error-404-isvc-ef1db-predictor.18c1ef3e788f07df namespace: kserve-ci-e2e-test resourceVersion: "21320" uid: af8883b5-d516-4bc2-bec4-549ab196bee9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 8e898d56-d325-429e-892b-1ea8143582cc kind: Event lastTimestamp: "2026-07-13T19:13:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:59Z" name: error-404-isvc-ef1db-predictor.18c1ef41f740b161 namespace: kserve-ci-e2e-test resourceVersion: "22033" uid: 7b549ac8-a455-4110-937a-882566aaecd1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21319" uid: 8e898d56-d325-429e-892b-1ea8143582cc kind: Event lastTimestamp: "2026-07-13T19:13:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:59Z" name: error-404-isvc-ef1db-predictor.18c1ef41f7a7e223 namespace: kserve-ci-e2e-test resourceVersion: "22034" uid: c822c353-f8b1-479a-b67e-b412a0bcb962 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 463c3fb9-0b18-4165-90f3-e1e65a88254c kind: Event lastTimestamp: "2026-07-13T19:13:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:41Z" name: error-404-isvc-ef1db.18c1ef3af9a1d24b namespace: kserve-ci-e2e-test resourceVersion: "21861" uid: c01a14c7-5e23-4b59-901d-501df5397867 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 463c3fb9-0b18-4165-90f3-e1e65a88254c kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: 'Failed to update status for InferenceService "error-404-isvc-ef1db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ef1db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:43Z" name: error-404-isvc-ef1db.18c1ef3af9efc5a2 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: af9e8bff-7214-40bf-b897-3c5114a03a5e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 463c3fb9-0b18-4165-90f3-e1e65a88254c kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ef1db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:44Z" name: error-404-isvc-ef1db.18c1ef3af9f00d62 namespace: kserve-ci-e2e-test resourceVersion: "21122" uid: 98f0cb51-2eda-4f7c-baee-ea5365b67b68 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 463c3fb9-0b18-4165-90f3-e1e65a88254c kind: Event lastTimestamp: "2026-07-13T19:13:41Z" message: InferenceService [error-404-isvc-ef1db] is Ready metadata: creationTimestamp: "2026-07-13T19:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:41Z" name: error-404-isvc-ef1db.18c1ef4868c0e41f namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: d70c9c87-2693-4159-b346-3b6dd67ac9a3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:00:24.563632Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14540" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:00:24Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee8ed0583f55 namespace: kserve-ci-e2e-test resourceVersion: "14546" uid: 556515e2-c52a-4738-8cf4-b1f875de1e94 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:24Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee8eda1303f8 namespace: kserve-ci-e2e-test resourceVersion: "14567" uid: ee5c553d-2a1f-46dc-a7e6-5f5f8e4cbafa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14549" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee8f0d8b70dd namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 8d93c245-aaf6-4000-a39c-a1bb24b595b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee8f0fb41d74 namespace: kserve-ci-e2e-test resourceVersion: "14688" uid: 55aa554a-9748-4642-ab64-f7e8df83ffeb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" in 5.279s (5.279s including waiting). Image size: 300570036 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:30Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee904a6543e2 namespace: kserve-ci-e2e-test resourceVersion: "14755" uid: 7da41d0e-2f49-4dd3-8327-a4160a9b7288 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:30Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:30Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee904d81b906 namespace: kserve-ci-e2e-test resourceVersion: "14756" uid: a0ae1fb1-03ca-4708-ac2e-be3d25b1d98a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:30Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:30Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee904df69b9e namespace: kserve-ci-e2e-test resourceVersion: "14757" uid: 6cf50513-a377-482f-8659-05108b446de9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:34Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" metadata: creationTimestamp: "2026-07-13T19:00:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:34Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee913364fbda namespace: kserve-ci-e2e-test resourceVersion: "14788" uid: 6a0b12ac-144b-4388-ab06-dd7bce600055 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:48Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" in 13.194s (13.194s including waiting). Image size: 1555978657 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:48Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee9445da00e3 namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: d6304055-3670-4e0f-b2bf-34b9ca8ea163 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:48Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee9449278ef2 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 501e6d26-c3ea-41b1-9182-6e88fa39c77b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:48Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee94499b595a namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: ec07422f-f4f5-4f6d-b187-aec89215d99f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:48Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T19:00:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:48Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee9449a82a4b namespace: kserve-ci-e2e-test resourceVersion: "14931" uid: 6fea5526-447e-4c92-bc4f-76644ce17591 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:50Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.357s (2.357s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:50Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee94d62a5501 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 65af7216-fb99-4cf4-9679-45e331ac450d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:50Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:50Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee94d9563eb3 namespace: kserve-ci-e2e-test resourceVersion: "14945" uid: 2bd599d6-13cf-4735-803f-10b733a8cba2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:00:50Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:50Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee94d9cc1950 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: f3be31e6-db90-4c1b-8f9e-3ef08abec55b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-13T19:00:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:02:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:00:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:46Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1ee94f2e142dd namespace: kserve-ci-e2e-test resourceVersion: "16341" uid: 8fdd23c6-2076-4942-a68a-2e5ead136538 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1eeaf6d626c4a namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: 488602c3-1e32-42b8-99b4-bedc18fb6ff4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1eeaf6d6387f1 namespace: kserve-ci-e2e-test resourceVersion: "16218" uid: 73a5f1bc-47f7-4464-9bf8-bfb8ed523d0f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs namespace: kserve-ci-e2e-test resourceVersion: "14543" uid: 0eaec241-64f4-4967-9032-3b2cde1b7898 kind: Event lastTimestamp: "2026-07-13T19:02:46Z" message: 'Readiness probe failed: Get "https://10.133.0.27:8643/healthz": dial tcp 10.133.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:46Z" name: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs.18c1eeaff534f25d namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 52d159a0-b048-48d3-aeda-83e20d3e296e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-858b494f9b namespace: kserve-ci-e2e-test resourceVersion: "14530" uid: ef898f4c-c318-4387-bdcb-c97c3c1423af kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-858b494f9b-fn5bs' metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: isvc-sklearn-graph-1-predictor-858b494f9b.18c1ee8ecf0c479b namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: 40032493-5a15-4a4b-a79d-359c326a5f41 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14529" uid: a1b81de9-c8c8-4dee-b8a1-bfed2234c781 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-858b494f9b from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: isvc-sklearn-graph-1-predictor.18c1ee8ec72f0ecc namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: 166e29eb-f7a1-4dd7-a3ed-a4bac6483fc6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 3ed4ecf2-8165-4380-a26a-be363547a8b4 kind: Event lastTimestamp: "2026-07-13T19:01:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:24Z" name: isvc-sklearn-graph-1-predictor.18c1ee924cee6302 namespace: kserve-ci-e2e-test resourceVersion: "15246" uid: 6ed7bdb1-4808-4fa5-ae9e-ac9cd562ed1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14536" uid: 3ed4ecf2-8165-4380-a26a-be363547a8b4 kind: Event lastTimestamp: "2026-07-13T19:01:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:24Z" name: isvc-sklearn-graph-1-predictor.18c1ee924d8de177 namespace: kserve-ci-e2e-test resourceVersion: "15247" uid: ecb1ee22-7ef1-453f-8225-ff02f2dc269e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 3ed4ecf2-8165-4380-a26a-be363547a8b4 kind: Event lastTimestamp: "2026-07-13T19:02:39Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:39Z" name: isvc-sklearn-graph-1-predictor.18c1eea04a07e649 namespace: kserve-ci-e2e-test resourceVersion: "16123" uid: 9f894f9e-a6c7-4d1f-946f-c1703dda8965 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14830" uid: 3ed4ecf2-8165-4380-a26a-be363547a8b4 kind: Event lastTimestamp: "2026-07-13T19:02:39Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:39Z" name: isvc-sklearn-graph-1-predictor.18c1eea04aabbc5f namespace: kserve-ci-e2e-test resourceVersion: "16124" uid: d3307cae-6163-4c78-9b22-74b2d803b86b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 6789c221-d97a-4551-bd7e-b445303abdbf kind: Event lastTimestamp: "2026-07-13T19:02:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:07Z" name: isvc-sklearn-graph-1.18c1ee8ecd38998d namespace: kserve-ci-e2e-test resourceVersion: "15817" uid: 863fa365-ffde-4bd7-acd6-0e043e41a91e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15812" uid: 6789c221-d97a-4551-bd7e-b445303abdbf kind: Event lastTimestamp: "2026-07-13T19:02:07Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-13T19:02:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:07Z" name: isvc-sklearn-graph-1.18c1eea6aac59123 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 2108e523-e9a3-4880-954b-7300f9b2d6a4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:00:25.168405Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14647" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh to ip-10-0-128-27.ec2.internal metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8ef464c8ee namespace: kserve-ci-e2e-test resourceVersion: "14653" uid: 4dd6669a-04f3-477f-a6d2-cfbe027ab4cb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8eff94cfe8 namespace: kserve-ci-e2e-test resourceVersion: "14672" uid: 864ee76f-b83c-4d44-871d-9c51fbb18fff reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:00:26Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8f31a2c2cb namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: ca09e9f1-a453-47b0-9c1d-4572ebe657a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:26Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8f3304f2df namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: f2562322-a4f0-4bfe-a23b-c12b912c155f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:26Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8f37167873 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 5f2861e4-a953-4202-b894-e516a3885c93 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:00:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:26Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee8f3786afbe namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: 8a5f41d2-3ea2-413b-b38d-7c6b046fcb18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" metadata: creationTimestamp: "2026-07-13T19:00:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:30Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee90245c69fa namespace: kserve-ci-e2e-test resourceVersion: "14746" uid: ec0e3391-9741-45c2-a079-cacc351e6a72 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1730" in 24.393s (24.393s including waiting). Image size: 1555978657 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee95d258fbce namespace: kserve-ci-e2e-test resourceVersion: "14982" uid: 90fbebd6-805e-4eb7-a3f0-b26f5fb52d5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee95d67cb349 namespace: kserve-ci-e2e-test resourceVersion: "14986" uid: 13545581-74f1-4825-803f-01655730c725 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee95d726ff34 namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 0f682cef-3a49-4073-916f-9fdc9de56e80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee95dc5467c8 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 3831227f-a79d-422c-a3ce-3069e57cb769 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.067s (3.067s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee969332a58e namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: e97a45df-60d5-4f75-b92c-7530f3c9202e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee9697595bb9 namespace: kserve-ci-e2e-test resourceVersion: "15027" uid: 8a8cb127-afbb-477f-88c7-3ff31ce7c6aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee9697cc9376 namespace: kserve-ci-e2e-test resourceVersion: "15029" uid: a2744055-fe3d-4ffd-b0ae-6b07c621adb5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:02:04Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:04Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1ee96b2ecaa89 namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: e8ba04a8-b649-467a-9cf8-eb5b7fd75d55 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1eeaf7699fa33 namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: eb29b889-14b8-4833-8a70-0f6b0bbbe070 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 4ca7e71e-2227-458a-9812-840ffccc956c kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh.18c1eeaf769a89f8 namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: ea971496-a188-49ff-94e5-2b023904fd34 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-d85b9cf8d namespace: kserve-ci-e2e-test resourceVersion: "14639" uid: 91406a73-d129-4d08-991c-506f6bcae5f1 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-d85b9cf8d-xxkwh' metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-2-predictor-d85b9cf8d.18c1ee8ef3db1861 namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: cd48beb8-1f43-481d-9b90-b77212d89cba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14636" uid: 6740cb6d-3f86-4c97-8486-9366ca338d09 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-d85b9cf8d from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:25Z" name: isvc-sklearn-graph-2-predictor.18c1ee8ef086598f namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: f52042b3-76f8-4da4-856e-70911ade275f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: ed1758fc-3dee-4518-8f8a-8482ff347afd kind: Event lastTimestamp: "2026-07-13T19:01:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:25Z" name: isvc-sklearn-graph-2-predictor.18c1ee926f00c716 namespace: kserve-ci-e2e-test resourceVersion: "15261" uid: 29e99134-6d87-4f6e-9185-48762700c256 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:40Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14638" uid: ed1758fc-3dee-4518-8f8a-8482ff347afd kind: Event lastTimestamp: "2026-07-13T19:01:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:25Z" name: isvc-sklearn-graph-2-predictor.18c1ee926f6e79a1 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: be5aaca8-634f-496e-ba00-d37cbdcf4077 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: ed1758fc-3dee-4518-8f8a-8482ff347afd kind: Event lastTimestamp: "2026-07-13T19:02:25Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:25Z" name: isvc-sklearn-graph-2-predictor.18c1ee996cd2ab2a namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: 31b9665a-56a7-4938-8ca3-0fc114369a3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14845" uid: ed1758fc-3dee-4518-8f8a-8482ff347afd kind: Event lastTimestamp: "2026-07-13T19:02:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:25Z" name: isvc-sklearn-graph-2-predictor.18c1ee996d427ad9 namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 686fb2b3-8b78-480e-9c39-0d6658245d19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: 747512e6-5987-411c-9217-8a7d31399fb3 kind: Event lastTimestamp: "2026-07-13T19:02:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:14Z" name: isvc-sklearn-graph-2.18c1ee8ef096e754 namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: ab634739-b4c7-4948-b9c0-e1b4735bc859 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15887" uid: 747512e6-5987-411c-9217-8a7d31399fb3 kind: Event lastTimestamp: "2026-07-13T19:02:14Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-13T19:02:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:14Z" name: isvc-sklearn-graph-2.18c1eea86ae7927b namespace: kserve-ci-e2e-test resourceVersion: "15889" uid: 022c7352-4120-451a-bfa7-d93002baaa2f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:00:24.951871Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-8z52k to ip-10-0-128-27.ec2.internal metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:00:24Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8ee77c4f47 namespace: kserve-ci-e2e-test resourceVersion: "14603" uid: 82a354a0-46af-4568-b741-4249a2053395 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14612" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:00:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8f012861e3 namespace: kserve-ci-e2e-test resourceVersion: "14673" uid: 1cf06f0b-9db9-4c9a-80f7-65e1f9ee068a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:38f70cb77701ee00560ecdde553dd84342a4d8f7cb0779aa5b508807d9895bf1" already present on machine metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8f02c0ea60 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: b1504f8d-9ef0-4e43-959c-d38b8a124968 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8f05a79b5e namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: be694057-72e5-424a-8868-90c7f25d9299 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8f06180d4c namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 4e8750e1-277f-450f-88ec-31693dd1c8fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:29Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-13T19:00:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee8fe833980c namespace: kserve-ci-e2e-test resourceVersion: "14736" uid: 1353fb4d-8ab0-4aab-8a49-0839979c5b06 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 25.64s (25.64s including waiting). Image size: 1305567991 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee95e0813d5e namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: 863687d7-6491-410b-95b5-b5fd48177171 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:00:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee95e6068447 namespace: kserve-ci-e2e-test resourceVersion: "14995" uid: f3f2871a-f053-4cbb-9e86-ea227aeedb4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee95e7132d37 namespace: kserve-ci-e2e-test resourceVersion: "14996" uid: d7466319-78a5-4265-935a-702f12bd0ac2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:55Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T19:00:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:55Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee95e7200c22 namespace: kserve-ci-e2e-test resourceVersion: "14997" uid: e8e9489f-77f1-4fe2-8dab-7212f03a27ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.9s (2.9s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee9694032d3f namespace: kserve-ci-e2e-test resourceVersion: "15026" uid: fee84a3b-3148-4629-8ea3-0ffc0db15157 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee969760b418 namespace: kserve-ci-e2e-test resourceVersion: "15028" uid: 4205bc26-4ec7-4010-b2d8-4d6bee1b1b0b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:00:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:00:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:57Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee9697d732fd namespace: kserve-ci-e2e-test resourceVersion: "15030" uid: 260385a6-80f6-4897-8429-554b6e7c5348 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:00:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:01:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:00:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1ee96b2d3e2a9 namespace: kserve-ci-e2e-test resourceVersion: "15453" uid: 955f99ef-1877-451b-bb6c-3f4ddac57e42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1eeaf6ff445a1 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: 4a83ecd0-3973-4e42-a9b7-89d195d5fe2b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8z52k namespace: kserve-ci-e2e-test resourceVersion: "14601" uid: df890e3d-097f-49f5-8c34-37048616e6d4 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8z52k.18c1eeaf6ff4a5eb namespace: kserve-ci-e2e-test resourceVersion: "16245" uid: af4afb23-4236-41ab-bc5f-ae9c3fbe424d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-27.ec2.internal source: component: kubelet host: ip-10-0-128-27.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 902a4774-fc15-4f75-a347-778827600573 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-8z52k' metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c1ee8ee6ed9115 namespace: kserve-ci-e2e-test resourceVersion: "14600" uid: 9ccf8cdd-efdb-4a21-9738-c24b13757a40 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14588" uid: c02dc2f1-9bbb-4e13-a417-cc91448179a7 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: isvc-xgboost-graph-predictor.18c1ee8ee261cfee namespace: kserve-ci-e2e-test resourceVersion: "14591" uid: 31f4cbaa-f279-47d9-82b6-8e2d38a7056a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 7a797d15-ad3a-43e0-83e4-6f5c5a4600c3 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:54Z" name: isvc-xgboost-graph-predictor.18c1ee9260c4b555 namespace: kserve-ci-e2e-test resourceVersion: "14989" uid: ab5b8c4b-24a1-4e20-a5bb-32bc0c64a5fd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: 7a797d15-ad3a-43e0-83e4-6f5c5a4600c3 kind: Event lastTimestamp: "2026-07-13T19:00:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:54Z" name: isvc-xgboost-graph-predictor.18c1ee92612a150a namespace: kserve-ci-e2e-test resourceVersion: "14991" uid: 0949922d-68b8-40cd-8adb-235b4896d808 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 7a797d15-ad3a-43e0-83e4-6f5c5a4600c3 kind: Event lastTimestamp: "2026-07-13T19:02:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:24Z" name: isvc-xgboost-graph-predictor.18c1ee995ea2276a namespace: kserve-ci-e2e-test resourceVersion: "15970" uid: fad299b0-0817-4319-a5c4-d18fe34e45c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:01:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14835" uid: 7a797d15-ad3a-43e0-83e4-6f5c5a4600c3 kind: Event lastTimestamp: "2026-07-13T19:02:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:24Z" name: isvc-xgboost-graph-predictor.18c1ee995eee1088 namespace: kserve-ci-e2e-test resourceVersion: "15971" uid: 9bbd2876-0ede-4d0a-a50e-a5976238f9b3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14568" uid: b4e83346-1ed7-488b-822a-31866eed8372 kind: Event lastTimestamp: "2026-07-13T19:01:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:54Z" name: isvc-xgboost-graph.18c1ee8ee27417e2 namespace: kserve-ci-e2e-test resourceVersion: "15544" uid: 9dffda8c-b273-47fb-90fe-be7e4fb61f64 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15542" uid: b4e83346-1ed7-488b-822a-31866eed8372 kind: Event lastTimestamp: "2026-07-13T19:01:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-13T19:01:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:54Z" name: isvc-xgboost-graph.18c1eea3c28af577 namespace: kserve-ci-e2e-test resourceVersion: "15543" uid: ff934d0e-9289-460a-a18f-896ca2c931b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:34.389635Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-667d9bc9cf-wnctj to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:34Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:34Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead0a93dc38 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 9339f0e8-6c9b-41ff-9594-73e33904c31d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:34Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:34Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:34Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead19c9ccaa namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: b0e40ad4-bf4d-4b2b-8fae-23272e236e19 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:35Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:35Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:02:35Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead47c67538 namespace: kserve-ci-e2e-test resourceVersion: "16084" uid: 6a64e589-55d9-4987-9e8f-15cf1f108685 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:35Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:35Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead49063be8 namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 26afb23f-177e-41bd-b4e1-7ca29ebc6768 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:35Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-13T19:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:35Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead4be9b528 namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: 16fdacb6-c874-4399-b4e7-5822ed68eb3e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:35Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-13T19:02:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:35Z" name: model-chainer-667d9bc9cf-wnctj.18c1eead4c58c09c namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 2f714ff4-927d-4998-b9f7-34e090b9b3d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:44Z" name: model-chainer-667d9bc9cf-wnctj.18c1eeaf6588ca1a namespace: kserve-ci-e2e-test resourceVersion: "16185" uid: 0289ce7d-09db-4bfe-abdb-fc06a639c672 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:02:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-667d9bc9cf-wnctj namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 02a034a2-9bd0-4a84-bc95-7b4465fceb03 kind: Event lastTimestamp: "2026-07-13T19:03:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:02:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:12Z" name: model-chainer-667d9bc9cf-wnctj.18c1eeb010b5a418 namespace: kserve-ci-e2e-test resourceVersion: "16603" uid: 92c90d9c-9752-4177-901c-9ed594bee2a9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:34Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-667d9bc9cf namespace: kserve-ci-e2e-test resourceVersion: "16059" uid: 406db4fd-5999-4b74-9f27-1b0f6ff739e5 kind: Event lastTimestamp: "2026-07-13T19:02:34Z" message: 'Created pod: model-chainer-667d9bc9cf-wnctj' metadata: creationTimestamp: "2026-07-13T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:34Z" name: model-chainer-667d9bc9cf.18c1eead0a3bd0f1 namespace: kserve-ci-e2e-test resourceVersion: "16064" uid: 8eb7fce9-f456-47d7-a839-9ac0bed2a941 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:34Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: e6838880-1c3c-4055-87b0-53ab20e4b759 kind: Event lastTimestamp: "2026-07-13T19:02:34Z" message: Scaled up replica set model-chainer-667d9bc9cf from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:34Z" name: model-chainer.18c1eead096f8eeb namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: 0b1e5480-5d9d-461e-9287-553c3d8db987 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:34Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16073" uid: 1314dfe8-ce2e-4446-b16c-c6f31f92e22c kind: Event lastTimestamp: "2026-07-13T19:02:34Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-13T19:02:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:34Z" name: model-chainer.18c1eead0ec51bd8 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: 0776325e-8604-4223-971e-6da88cce8f2d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:03:44.721281Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-3a16a-8677b985dd-w6wzw to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:03:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:03:44Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebd6aaba729 namespace: kserve-ci-e2e-test resourceVersion: "16894" uid: 2a5ece4a-5c2d-42e3-85d8-511bf571f7e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-3a16a-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:44Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebd76d0209f namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: ad16b235-2c8e-406b-85fb-179c34df8548 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:45Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16898" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:03:45Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:03:45Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebda7e4b7bc namespace: kserve-ci-e2e-test resourceVersion: "16916" uid: dca71c2c-5a15-41d4-b4a5-0a69315d2696 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a16a} kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:03:45Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:45Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebda926652b namespace: kserve-ci-e2e-test resourceVersion: "16919" uid: 6006dd49-3268-41f6-aacb-e611d5c558ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a16a} kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:03:45Z" message: 'Created container: sequence-graph-3a16a' metadata: creationTimestamp: "2026-07-13T19:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:45Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebdac0b8f01 namespace: kserve-ci-e2e-test resourceVersion: "16921" uid: 571e60b8-0c00-4fdc-8a63-2475924db66b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a16a} kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:03:45Z" message: Started container sequence-graph-3a16a metadata: creationTimestamp: "2026-07-13T19:03:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:45Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1eebdac76560f namespace: kserve-ci-e2e-test resourceVersion: "16922" uid: a16eb4bb-3a1f-4f64-be38-9b2bf1dd1c28 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a16a} kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Stopping container sequence-graph-3a16a metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:59Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1ef3099883426 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: 31a5f1f0-bb0b-40d5-8aa5-918a2a7d647f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:12:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-3a16a} kind: Pod name: sequence-graph-3a16a-8677b985dd-w6wzw namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 9a9a8421-cbda-4fe6-ab1f-7caae57d3179 kind: Event lastTimestamp: "2026-07-13T19:12:27Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:27Z" name: sequence-graph-3a16a-8677b985dd-w6wzw.18c1ef315c0e543d namespace: kserve-ci-e2e-test resourceVersion: "20866" uid: bbf5d620-cc67-48d3-8030-93771ad2b9e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-3a16a-8677b985dd namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: 3349d107-b939-4fc7-8577-0012cb0d084e kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: 'Created pod: sequence-graph-3a16a-8677b985dd-w6wzw' metadata: creationTimestamp: "2026-07-13T19:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: sequence-graph-3a16a-8677b985dd.18c1eebd6a295fe2 namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: c20edf36-4819-404e-90f3-43621173089a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: 9f0ca656-8c2c-45c7-9cf0-c69335c86ff4 kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: Scaled up replica set sequence-graph-3a16a-8677b985dd from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:44Z" name: sequence-graph-3a16a.18c1eebd69402641 namespace: kserve-ci-e2e-test resourceVersion: "16884" uid: f6b17909-f30f-4324-ab44-05ce49a86ab5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16900" uid: 23c4c10e-dc20-4a21-8498-33221ccd417f kind: Event lastTimestamp: "2026-07-13T19:03:44Z" message: InferenceGraph [sequence-graph-3a16a] is Ready metadata: creationTimestamp: "2026-07-13T19:03:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:44Z" name: sequence-graph-3a16a.18c1eebd6f4dbaa1 namespace: kserve-ci-e2e-test resourceVersion: "16902" uid: 5e96ed88-3940-44f5-895f-03f134a14276 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: a4bb7180-1783-4706-80eb-c68375d5dcbe kind: Event lastTimestamp: "2026-07-13T19:04:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:14Z" name: sequence-graph-3a16a.18c1eec0e7ca24b8 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 01b40269-f539-4f3d-ac99-873901e4912a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:03:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16882" uid: a4bb7180-1783-4706-80eb-c68375d5dcbe kind: Event lastTimestamp: "2026-07-13T19:04:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:04:14Z" name: sequence-graph-3a16a.18c1eec0e830af61 namespace: kserve-ci-e2e-test resourceVersion: "17145" uid: b81fbc04-b090-42b0-9bd6-d7f7a5a401ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:14:30.006247Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22294" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-a68dc-67db58c45f-v9xr8 to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1ef53a8a6d722 namespace: kserve-ci-e2e-test resourceVersion: "22299" uid: d1fe2047-c8c6-447c-8ddd-621cb6dd55a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:30Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22304" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:14:30Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1ef53c2721bb9 namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: 83e35cf8-b31a-4de4-8107-21b5dcb151b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a68dc} kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:14:30Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1ef53c4308b01 namespace: kserve-ci-e2e-test resourceVersion: "22315" uid: d8fdb1a0-88bf-4c42-99bd-bcc6ac962be1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a68dc} kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:14:30Z" message: 'Created container: sequence-graph-a68dc' metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1ef53c72bd1c0 namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: 4c3f0fc5-2f44-448c-bbf1-724813d3e083 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a68dc} kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:14:30Z" message: Started container sequence-graph-a68dc metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1ef53c79ee624 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: 3ce5e0bf-a308-4131-846d-b945a98ffdd8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a68dc} kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Stopping container sequence-graph-a68dc metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1efc6cdf11f55 namespace: kserve-ci-e2e-test resourceVersion: "26188" uid: a6fa28db-f58d-44c3-a13b-1c167741c625 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-a68dc} kind: Pod name: sequence-graph-a68dc-67db58c45f-v9xr8 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 78f763e5-4ef5-4fc3-bb39-73c0a36cb3e8 kind: Event lastTimestamp: "2026-07-13T19:23:12Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:12Z" name: sequence-graph-a68dc-67db58c45f-v9xr8.18c1efc778d73365 namespace: kserve-ci-e2e-test resourceVersion: "26558" uid: cf7da7f9-54fa-4b56-96ad-551ab3ce6aba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-a68dc-67db58c45f namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 668a6d81-0506-4a54-816b-8a11750d0a7c kind: Event lastTimestamp: "2026-07-13T19:14:29Z" message: 'Created pod: sequence-graph-a68dc-67db58c45f-v9xr8' metadata: creationTimestamp: "2026-07-13T19:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:29Z" name: sequence-graph-a68dc-67db58c45f.18c1ef53a7ddf485 namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: bb08e639-1fb4-4f47-a192-47e8984b8cd8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 07248a59-a63f-4987-a656-ec0fb7b8c348 kind: Event lastTimestamp: "2026-07-13T19:14:29Z" message: Scaled up replica set sequence-graph-a68dc-67db58c45f from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:14:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:29Z" name: sequence-graph-a68dc.18c1ef53a6fe1778 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: 41ec78e6-85d8-49d3-876a-b6a3599f4b84 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22306" uid: af40232c-12b4-4b8e-9955-e899bcb0d904 kind: Event lastTimestamp: "2026-07-13T19:14:30Z" message: InferenceGraph [sequence-graph-a68dc] is Ready metadata: creationTimestamp: "2026-07-13T19:14:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:30Z" name: sequence-graph-a68dc.18c1ef53ace70ac6 namespace: kserve-ci-e2e-test resourceVersion: "22308" uid: 54960b85-68c8-431d-9db4-bd7fae400647 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 37884fb8-8e63-43a9-a2eb-09d9b4ad753b kind: Event lastTimestamp: "2026-07-13T19:14:44Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:44Z" name: sequence-graph-a68dc.18c1ef5725a747ca namespace: kserve-ci-e2e-test resourceVersion: "22434" uid: f97b41b4-851b-4f95-ad2a-5669e26cdac1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22290" uid: 37884fb8-8e63-43a9-a2eb-09d9b4ad753b kind: Event lastTimestamp: "2026-07-13T19:14:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:14:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:44Z" name: sequence-graph-a68dc.18c1ef572620f438 namespace: kserve-ci-e2e-test resourceVersion: "22436" uid: ec38d1fb-0f1a-43f7-84d2-b6d314fd64fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 37884fb8-8e63-43a9-a2eb-09d9b4ad753b kind: Event lastTimestamp: "2026-07-13T19:15:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:15:00Z" name: sequence-graph-a68dc.18c1ef5aa4e64c75 namespace: kserve-ci-e2e-test resourceVersion: "22540" uid: aac7dc43-dacb-4db5-a42a-a745da7ba708 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:15:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22435" uid: 37884fb8-8e63-43a9-a2eb-09d9b4ad753b kind: Event lastTimestamp: "2026-07-13T19:15:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:15:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:15:00Z" name: sequence-graph-a68dc.18c1ef5aa5571247 namespace: kserve-ci-e2e-test resourceVersion: "22542" uid: f217fa05-e640-42aa-a5ea-988187c9059f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:13:09.683052Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e56dd-5454f86ccd-qjtd2 to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:13:09Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef40f5040de8 namespace: kserve-ci-e2e-test resourceVersion: "21413" uid: 9016d298-5586-4ff1-864e-e790326e24c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:10Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:10Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:13:10Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef410e908799 namespace: kserve-ci-e2e-test resourceVersion: "21427" uid: 0ed56ae3-aecf-4386-8574-6287ad9d96ab reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e56dd} kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:10Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:10Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef410fc6aa06 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 83e3a6ba-d3f6-4d31-a182-464d50c57c71 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e56dd} kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:10Z" message: 'Created container: sequence-graph-e56dd' metadata: creationTimestamp: "2026-07-13T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:10Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef4113b27f33 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 8dc516b2-91d2-4eda-80da-407b3c4163a9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e56dd} kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:10Z" message: Started container sequence-graph-e56dd metadata: creationTimestamp: "2026-07-13T19:13:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:10Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef411424482d namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: f56153c1-20ac-47f0-bb5b-7be691f00677 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e56dd} kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Stopping container sequence-graph-e56dd metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:19Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef434df43151 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: b25aa0a8-ad1f-4546-b741-d4853775148c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e56dd} kind: Pod name: sequence-graph-e56dd-5454f86ccd-qjtd2 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: 163b5bef-4979-4cf4-bf62-a69a29e99a80 kind: Event lastTimestamp: "2026-07-13T19:13:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:47Z" name: sequence-graph-e56dd-5454f86ccd-qjtd2.18c1ef43d788f3bb namespace: kserve-ci-e2e-test resourceVersion: "21906" uid: 30b6cf93-13e9-450d-a717-7089f5b670f9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e56dd-5454f86ccd namespace: kserve-ci-e2e-test resourceVersion: "21404" uid: 4ddb0d28-08ad-4aeb-ba56-2429308b8fee kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: 'Created pod: sequence-graph-e56dd-5454f86ccd-qjtd2' metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:09Z" name: sequence-graph-e56dd-5454f86ccd.18c1ef40f3b13833 namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: f3ab4207-9455-4229-913a-7f46404dae7b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e56dd namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 545d6ef7-9faf-48b5-8874-a2c8eccd8f1f kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: Scaled up replica set sequence-graph-e56dd-5454f86ccd from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:09Z" name: sequence-graph-e56dd.18c1ef40f2297079 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: c6741a5d-2d91-42b4-8122-b93652603c63 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:09Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e56dd namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: 3b1a049b-f66b-40ec-857d-ea8d8ece639a kind: Event lastTimestamp: "2026-07-13T19:13:09Z" message: InferenceGraph [sequence-graph-e56dd] is Ready metadata: creationTimestamp: "2026-07-13T19:13:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:09Z" name: sequence-graph-e56dd.18c1ef40fb096162 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: e5adf155-265b-4dbd-9a4e-4198aa1a9f74 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:24:39.315701Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27510" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-08823-79fbf9cdc-w6qzn to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1efe186533568 namespace: kserve-ci-e2e-test resourceVersion: "27515" uid: c1a7480e-81d6-4fab-9dc5-3397a9e0f916 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27519" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: Add eth0 [10.133.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1efe19fabf85e namespace: kserve-ci-e2e-test resourceVersion: "27529" uid: 20f18b62-d50c-479b-86b9-125fe249898b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-08823} kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1efe1a120cc26 namespace: kserve-ci-e2e-test resourceVersion: "27531" uid: 605e4434-8f59-44bd-b035-4047c627d24a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-08823} kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: 'Created container: splitter-graph-08823' metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1efe1a441322b namespace: kserve-ci-e2e-test resourceVersion: "27533" uid: 07f36470-514c-4b39-928b-994418dea93b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-08823} kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: Started container splitter-graph-08823 metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1efe1a4b27af3 namespace: kserve-ci-e2e-test resourceVersion: "27534" uid: ea0016e7-6cb6-4b01-b5c1-87b1e58c4f41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-08823} kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:32:53Z" message: Stopping container splitter-graph-08823 metadata: creationTimestamp: "2026-07-13T19:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:53Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1f054b15905b7 namespace: kserve-ci-e2e-test resourceVersion: "30971" uid: f1e26a59-9a2d-4fe0-abd3-3d8b5c305815 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:32:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-08823} kind: Pod name: splitter-graph-08823-79fbf9cdc-w6qzn namespace: kserve-ci-e2e-test resourceVersion: "27512" uid: 40973dff-2d0b-4bf8-98f2-5d680018d5cd kind: Event lastTimestamp: "2026-07-13T19:33:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:32:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:33:20Z" name: splitter-graph-08823-79fbf9cdc-w6qzn.18c1f055296b5e00 namespace: kserve-ci-e2e-test resourceVersion: "31202" uid: 5a36535b-b447-4a3c-afda-9db8f7e618a5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-08823-79fbf9cdc namespace: kserve-ci-e2e-test resourceVersion: "27507" uid: 8837f016-f6c0-43ed-9e08-6f365978605d kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: 'Created pod: splitter-graph-08823-79fbf9cdc-w6qzn' metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823-79fbf9cdc.18c1efe185db828f namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: 7dffa5c5-0261-43a8-8fc5-4a8f59d95749 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-08823 namespace: kserve-ci-e2e-test resourceVersion: "27505" uid: d222de9e-0a0b-451a-92f0-7c94a489e8da kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: Scaled up replica set splitter-graph-08823-79fbf9cdc from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823.18c1efe184ff8252 namespace: kserve-ci-e2e-test resourceVersion: "27508" uid: 96d9b560-4007-4668-986e-5c83058e93b4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-08823 namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 1bbc520b-d927-41aa-b6ea-4435914b75b7 kind: Event lastTimestamp: "2026-07-13T19:24:39Z" message: InferenceGraph [splitter-graph-08823] is Ready metadata: creationTimestamp: "2026-07-13T19:24:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:24:39Z" name: splitter-graph-08823.18c1efe18ab6b480 namespace: kserve-ci-e2e-test resourceVersion: "27525" uid: e1407ca6-86bf-4ac2-a975-50ff5006c2e5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-08823 namespace: kserve-ci-e2e-test resourceVersion: "27506" uid: 2d6deb8b-0a46-4278-962d-3786b9a7005a kind: Event lastTimestamp: "2026-07-13T19:25:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:25:09Z" name: splitter-graph-08823.18c1efe50388fbdf namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: a04f414d-bd0f-4247-bfae-1483e3862d29 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:24:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-08823 namespace: kserve-ci-e2e-test resourceVersion: "27506" uid: 2d6deb8b-0a46-4278-962d-3786b9a7005a kind: Event lastTimestamp: "2026-07-13T19:25:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:24:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:25:09Z" name: splitter-graph-08823.18c1efe503f32cd0 namespace: kserve-ci-e2e-test resourceVersion: "27763" uid: f9a09edf-2745-491b-80e4-4ac268b0f7ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:23:18.979727Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26629" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-b5afb-57fdcdd6c9-bwbw4 to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:23:18Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:23:18Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efced1ed9366 namespace: kserve-ci-e2e-test resourceVersion: "26634" uid: cdf0571f-33b9-4fe3-bfc7-dd45a7604619 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:19Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26638" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:19Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:23:19Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efceeb90a724 namespace: kserve-ci-e2e-test resourceVersion: "26647" uid: d3e4c051-f005-4306-a5e8-c3358fd0504e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5afb} kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:19Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:19Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efceece8292e namespace: kserve-ci-e2e-test resourceVersion: "26649" uid: 1908bd05-995e-4af5-9047-ecbcf8668851 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5afb} kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:19Z" message: 'Created container: splitter-graph-b5afb' metadata: creationTimestamp: "2026-07-13T19:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:19Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efceef995b06 namespace: kserve-ci-e2e-test resourceVersion: "26650" uid: 96e3c8d7-d127-4481-b003-425fbcbf63d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5afb} kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:19Z" message: Started container splitter-graph-b5afb metadata: creationTimestamp: "2026-07-13T19:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:19Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efcef00a2ec4 namespace: kserve-ci-e2e-test resourceVersion: "26651" uid: 0c9be427-c0a6-4d43-a118-bfd51c1f0249 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5afb} kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Stopping container splitter-graph-b5afb metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efd129bb4762 namespace: kserve-ci-e2e-test resourceVersion: "26742" uid: bf8b0327-7893-4ecd-bbab-14d76ed142a0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-b5afb} kind: Pod name: splitter-graph-b5afb-57fdcdd6c9-bwbw4 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: be353345-69fa-4cb7-a619-a0c29ba18c16 kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:55Z" name: splitter-graph-b5afb-57fdcdd6c9-bwbw4.18c1efd18823d07c namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: b01bfaec-5504-46b7-b9c4-82e537ba7029 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:18Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-b5afb-57fdcdd6c9 namespace: kserve-ci-e2e-test resourceVersion: "26626" uid: 58ddc3a7-6eb3-4ea6-98e3-b165f353c089 kind: Event lastTimestamp: "2026-07-13T19:23:18Z" message: 'Created pod: splitter-graph-b5afb-57fdcdd6c9-bwbw4' metadata: creationTimestamp: "2026-07-13T19:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:18Z" name: splitter-graph-b5afb-57fdcdd6c9.18c1efced1617025 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: 37c414de-550c-45b3-9b2e-176479bc74d7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:18Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-b5afb namespace: kserve-ci-e2e-test resourceVersion: "26624" uid: 00d6b146-6684-4c52-9297-ebf0f4952443 kind: Event lastTimestamp: "2026-07-13T19:23:18Z" message: Scaled up replica set splitter-graph-b5afb-57fdcdd6c9 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:23:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:18Z" name: splitter-graph-b5afb.18c1efced02c24e4 namespace: kserve-ci-e2e-test resourceVersion: "26627" uid: 395e88b6-4914-4f17-884f-bd64f07154de reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:19Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-b5afb namespace: kserve-ci-e2e-test resourceVersion: "26640" uid: 3062d6e2-9ac2-4e75-9c3d-84d2acf9f220 kind: Event lastTimestamp: "2026-07-13T19:23:19Z" message: InferenceGraph [splitter-graph-b5afb] is Ready metadata: creationTimestamp: "2026-07-13T19:23:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:19Z" name: splitter-graph-b5afb.18c1efced5c4e677 namespace: kserve-ci-e2e-test resourceVersion: "26642" uid: 5d652bff-bbcb-4a73-a171-be34e053ce5d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:23:29.229019Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26792" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-08823-predictor-67685c4f88-qkbp9 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd134d56209 namespace: kserve-ci-e2e-test resourceVersion: "26800" uid: fc751884-1a5a-4c2c-882f-b53c64fb3032 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-08823-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd13ebffc84 namespace: kserve-ci-e2e-test resourceVersion: "26833" uid: ffa0c1d4-7949-4756-ae03-c89b54adfc38 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26805" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd17299c970 namespace: kserve-ci-e2e-test resourceVersion: "26868" uid: 11d421b5-2fed-48c0-8ea1-e9eafd7a0479 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd17426b723 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: ff1ed2c7-e520-46c7-8f16-05960dee79c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd1777a01d4 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 9f721c87-8c41-4013-9cc1-d66d3149be12 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd177eebcc2 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 9e1a3b77-9322-40c7-a33e-b8822feeece0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd177f9f1a6 namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: c27a7cb7-d1d9-4651-947f-9ba7caf0a0da reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd17b4d2cf8 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: f0ae4cb1-a427-41bc-b4e6-711f4e7beddf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:23:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:23:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:30Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd17bc358e5 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 8be88112-c850-457c-8b25-d99d65611a63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:23:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:24:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:23:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:24:18Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1efd1a9c884ca namespace: kserve-ci-e2e-test resourceVersion: "27334" uid: 27851d9c-2c77-4959-82bf-3ceec2449b94 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:32:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:54Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1f054b857ed01 namespace: kserve-ci-e2e-test resourceVersion: "30988" uid: 57bb9dd8-9fb9-400e-8be2-3a7a246c8d08 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-08823-predictor-67685c4f88-qkbp9 namespace: kserve-ci-e2e-test resourceVersion: "26796" uid: dbf1dcca-102f-4118-abfa-66fc5a1ca204 kind: Event lastTimestamp: "2026-07-13T19:32:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:32:54Z" name: success-200-isvc-08823-predictor-67685c4f88-qkbp9.18c1f054b8588024 namespace: kserve-ci-e2e-test resourceVersion: "30994" uid: 2f136d75-01bb-4d8b-8e28-ac770d8a0c7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-08823-predictor-67685c4f88 namespace: kserve-ci-e2e-test resourceVersion: "26778" uid: d5443acf-229c-4888-8f0b-e4f85d23a1e2 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Created pod: success-200-isvc-08823-predictor-67685c4f88-qkbp9' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823-predictor-67685c4f88.18c1efd13440a32a namespace: kserve-ci-e2e-test resourceVersion: "26795" uid: 091a670a-d750-42c2-95e7-230948bb839c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26771" uid: f4d87c09-06c1-4329-a3fe-1572bfed1ce3 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Scaled up replica set success-200-isvc-08823-predictor-67685c4f88 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823-predictor.18c1efd1326e5a35 namespace: kserve-ci-e2e-test resourceVersion: "26782" uid: 2ee28478-6d23-4f60-8ffc-1e1e92b98d75 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:23:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: 74a00289-f9bb-4755-980c-8af9d2915398 kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:23:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: success-200-isvc-08823-predictor.18c1efd4b16a7f5b namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: 0889c9b5-4118-4bc5-8c62-cc3a865e54c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:23:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "26776" uid: 74a00289-f9bb-4755-980c-8af9d2915398 kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:23:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: success-200-isvc-08823-predictor.18c1efd4b1db0fef namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: 57f852e7-4f9c-4678-8023-a98723eb09c8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:24:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 74a00289-f9bb-4755-980c-8af9d2915398 kind: Event lastTimestamp: "2026-07-13T19:24:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:59Z" name: success-200-isvc-08823-predictor.18c1efdbaf601847 namespace: kserve-ci-e2e-test resourceVersion: "27690" uid: 68122de4-8199-43fa-af90-6f30baa9bdb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:24:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-08823-predictor namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: 74a00289-f9bb-4755-980c-8af9d2915398 kind: Event lastTimestamp: "2026-07-13T19:24:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:59Z" name: success-200-isvc-08823-predictor.18c1efdbafd4c1fc namespace: kserve-ci-e2e-test resourceVersion: "27691" uid: faef85d0-cccc-4f9b-9313-1a306798a3e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: fcab5657-a779-40e5-8eb7-ce092b27ab19 kind: Event lastTimestamp: "2026-07-13T19:24:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:24:28Z" name: success-200-isvc-08823.18c1efd132d64f95 namespace: kserve-ci-e2e-test resourceVersion: "27417" uid: e8f4ff99-d27c-4f48-b0c4-960c0f15c278 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: fcab5657-a779-40e5-8eb7-ce092b27ab19 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'Failed to update status for InferenceService "success-200-isvc-08823": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-08823": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823.18c1efd1335dd636 namespace: kserve-ci-e2e-test resourceVersion: "26789" uid: c124eac2-13a7-4ad1-9730-2208f90ea0c0 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: fcab5657-a779-40e5-8eb7-ce092b27ab19 kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-08823": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-08823.18c1efd1335e3af9 namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: 550982d4-b06f-4121-ba21-cfa0412bbc07 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:24:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-08823 namespace: kserve-ci-e2e-test resourceVersion: "27414" uid: fcab5657-a779-40e5-8eb7-ce092b27ab19 kind: Event lastTimestamp: "2026-07-13T19:24:28Z" message: InferenceService [success-200-isvc-08823] is Ready metadata: creationTimestamp: "2026-07-13T19:24:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:24:28Z" name: success-200-isvc-08823.18c1efdef56e6467 namespace: kserve-ci-e2e-test resourceVersion: "27415" uid: a49850bd-f96b-4bea-8213-0da2596c6c62 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:00:24.767288Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14569" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-19366-predictor-6b66d49548-wrwt5 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:00:24Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee8edc7bdf8d namespace: kserve-ci-e2e-test resourceVersion: "14573" uid: 7e3eb5c2-d407-4336-9a29-185266d93749 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Add eth0 [10.134.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:00:25Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee8ef6936ba8 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 04cd877e-abbc-425a-97ae-49baec9a86d0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:25Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" metadata: creationTimestamp: "2026-07-13T19:00:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:25Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee8ef7eeb661 namespace: kserve-ci-e2e-test resourceVersion: "14664" uid: 556f9530-a950-4535-95df-f94a5c6d184d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" in 16.193s (16.193s including waiting). Image size: 1337530974 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee92bd213612 namespace: kserve-ci-e2e-test resourceVersion: "14857" uid: 1ec06313-0bab-4dca-bafa-e97a95a94657 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee92c0d7e820 namespace: kserve-ci-e2e-test resourceVersion: "14860" uid: c9efddea-7a44-4e37-b4d2-6eff5234c481 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee92c16bb5d9 namespace: kserve-ci-e2e-test resourceVersion: "14861" uid: 37333d22-baf2-4fcf-b5a1-5e92e3009227 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-13T19:00:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:41Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee92c179e2cf namespace: kserve-ci-e2e-test resourceVersion: "14864" uid: 69fed5a7-377f-402f-8da8-32d1d06093ce reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.576s (2.576s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee935b11e3ed namespace: kserve-ci-e2e-test resourceVersion: "14883" uid: c31ac603-c3fa-4a23-bc43-f1f419ac61c1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee935efb4941 namespace: kserve-ci-e2e-test resourceVersion: "14885" uid: 4ca0741c-fe46-48a5-9311-15fa90925197 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:00:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:00:44Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee935f7542f4 namespace: kserve-ci-e2e-test resourceVersion: "14887" uid: 4a50cb3d-1d55-44fa-a5e8-51f6aba2f9d5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:00:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:02:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:00:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:00Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1ee93828c1aea namespace: kserve-ci-e2e-test resourceVersion: "15744" uid: 61b6b2b6-1995-4b86-a652-97c1f5eca721 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1eea4b69834c9 namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: 8f476bfe-5f80-4695-8955-6c3ca80d989f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1eea4b69956bf namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 1bc09dc6-b380-4e6d-9357-e981999fe8cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-19366-predictor-6b66d49548-wrwt5 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 6e489e9b-a37d-4a7f-a8f4-8f53153883c3 kind: Event lastTimestamp: "2026-07-13T19:02:00Z" message: 'Readiness probe failed: Get "https://10.134.0.22:8643/healthz": dial tcp 10.134.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:00Z" name: success-200-isvc-19366-predictor-6b66d49548-wrwt5.18c1eea534c003fe namespace: kserve-ci-e2e-test resourceVersion: "15743" uid: 626a9ed5-fcd1-4233-bb21-eac69a6663a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-19366-predictor-6b66d49548 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: fc401ee3-61f1-486f-9b71-298026fe1ee5 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: 'Created pod: success-200-isvc-19366-predictor-6b66d49548-wrwt5' metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: success-200-isvc-19366-predictor-6b66d49548.18c1ee8edb7b8d52 namespace: kserve-ci-e2e-test resourceVersion: "14570" uid: b12099c4-bf98-499d-8375-0260f183bf84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14555" uid: 4782bead-e2c2-4bed-a34c-d2603e7b0c68 kind: Event lastTimestamp: "2026-07-13T19:00:24Z" message: Scaled up replica set success-200-isvc-19366-predictor-6b66d49548 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:00:24Z" name: success-200-isvc-19366-predictor.18c1ee8ed633c869 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 70f11e51-8ae4-4b15-85e9-26e3f4b155a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 5007ac81-e53b-4040-84c0-bc9c2a74b080 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: success-200-isvc-19366-predictor.18c1ee9254f1e4df namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: 6714e28f-13d4-4f6a-bab8-ffab1c2ca91f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:00:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 5007ac81-e53b-4040-84c0-bc9c2a74b080 kind: Event lastTimestamp: "2026-07-13T19:01:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:09Z" name: success-200-isvc-19366-predictor.18c1ee92556d2fa6 namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: b71ae8e1-e325-4157-8034-8ac1b49ec1bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: 5007ac81-e53b-4040-84c0-bc9c2a74b080 kind: Event lastTimestamp: "2026-07-13T19:01:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:54Z" name: success-200-isvc-19366-predictor.18c1ee9cd19727c1 namespace: kserve-ci-e2e-test resourceVersion: "15548" uid: 6a467170-1001-4901-89f2-cfe550c6fdd9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:01:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-19366-predictor namespace: kserve-ci-e2e-test resourceVersion: "14833" uid: 5007ac81-e53b-4040-84c0-bc9c2a74b080 kind: Event lastTimestamp: "2026-07-13T19:01:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:01:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:54Z" name: success-200-isvc-19366-predictor.18c1ee9cd1f9715d namespace: kserve-ci-e2e-test resourceVersion: "15549" uid: a50b5da4-1296-4ee1-b89f-99830f97884f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:00:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19366 namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 128ab57d-7518-46da-a39c-cde7aae457b9 kind: Event lastTimestamp: "2026-07-13T19:01:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:00:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:30Z" name: success-200-isvc-19366.18c1ee8ed66c316c namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: c3287129-f5d5-446c-bf8a-267d4bd26617 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-19366 namespace: kserve-ci-e2e-test resourceVersion: "15306" uid: 128ab57d-7518-46da-a39c-cde7aae457b9 kind: Event lastTimestamp: "2026-07-13T19:01:30Z" message: InferenceService [success-200-isvc-19366] is Ready metadata: creationTimestamp: "2026-07-13T19:01:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:30Z" name: success-200-isvc-19366.18c1ee9e3dd8d144 namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 242b0564-16f6-4fd0-87da-9646aad618a9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:02:44.676827Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3a16a-predictor-68664fb5d6-5cmst to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:02:44Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf6fbdff99 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: ca41a571-bff9-43ae-a76d-0b52fe044b2a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16248" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Add eth0 [10.134.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf89b4dc0b namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 865780a3-f6b3-4fee-90e3-a2289012583f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf8afb56b2 namespace: kserve-ci-e2e-test resourceVersion: "16306" uid: c1458118-6486-4a8b-8211-3f89a523c945 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf8e94f865 namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: d6a903e8-af01-41ba-9f4a-a47fb4e7215c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf8f7d934f namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: e2d8b7ca-01eb-453a-af21-084c718eee37 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf8f884aa2 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 8bb30172-0096-4a60-bf7d-68b21640ea17 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf92ca54d3 namespace: kserve-ci-e2e-test resourceVersion: "16313" uid: 8c7fa18e-8f67-4e16-a569-50310db2c08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:02:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:02:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:45Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeaf93425888 namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: b1f832f5-5ba4-465d-84dd-8664ae1b16bc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:02:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:12:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:02Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1eeafcb5fe535 namespace: kserve-ci-e2e-test resourceVersion: "20633" uid: 56d10e70-3ebc-47d0-8b79-03043e894c44 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1ef30a3d6a28f namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: e0367134-4402-4603-a0a4-6dc92a80d833 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1ef30a3d745e7 namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: 09ef5bf6-5618-4a15-a4db-7ee2d0ecd888 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 2947986d-5361-4e23-908d-c270c4637661 kind: Event lastTimestamp: "2026-07-13T19:12:02Z" message: 'Readiness probe failed: Get "https://10.134.0.26:8643/healthz": dial tcp 10.134.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:02Z" name: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst.18c1ef313fc4a8be namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: bb47b63b-6a53-4aa0-9932-82b572949899 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3a16a-predictor-68664fb5d6 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 8b32b4e6-bce0-4b89-b2c4-36ffc6b401fd kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: 'Created pod: success-200-isvc-3a16a-predictor-68664fb5d6-5cmst' metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:44Z" name: success-200-isvc-3a16a-predictor-68664fb5d6.18c1eeaf6f4ea24a namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: fa9bb74e-0e97-4a36-8644-10a2e209a9e0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16209" uid: 518e4522-3a8b-4d5b-8719-14a46c268296 kind: Event lastTimestamp: "2026-07-13T19:02:44Z" message: Scaled up replica set success-200-isvc-3a16a-predictor-68664fb5d6 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:44Z" name: success-200-isvc-3a16a-predictor.18c1eeaf6d844c9d namespace: kserve-ci-e2e-test resourceVersion: "16215" uid: 54968143-7c2d-4f8f-bb01-fdb7b7c70310 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: ad169339-99a9-4704-91bf-ca4914aa8701 kind: Event lastTimestamp: "2026-07-13T19:03:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:14Z" name: success-200-isvc-3a16a-predictor.18c1eeb2ec373fd3 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 002cc195-ccee-4c4f-8123-8470df1a4bd6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16211" uid: ad169339-99a9-4704-91bf-ca4914aa8701 kind: Event lastTimestamp: "2026-07-13T19:03:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:14Z" name: success-200-isvc-3a16a-predictor.18c1eeb2ece6d3e1 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 0ebecb5d-1ac7-4dba-bb40-3881e8b1749f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ad169339-99a9-4704-91bf-ca4914aa8701 kind: Event lastTimestamp: "2026-07-13T19:03:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:59Z" name: success-200-isvc-3a16a-predictor.18c1eeb9eaca36d1 namespace: kserve-ci-e2e-test resourceVersion: "17032" uid: 0f26509f-0caa-4028-8a7b-ee7f1e6b8e88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:03:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3a16a-predictor namespace: kserve-ci-e2e-test resourceVersion: "16474" uid: ad169339-99a9-4704-91bf-ca4914aa8701 kind: Event lastTimestamp: "2026-07-13T19:03:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:59Z" name: success-200-isvc-3a16a-predictor.18c1eeb9eb1bbefe namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 8bb43deb-97b3-4dfe-b3cd-8d0be5b83229 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:02:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16191" uid: c94d0f5b-eea8-4556-a991-ac47458d22ad kind: Event lastTimestamp: "2026-07-13T19:03:32Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:02:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:32Z" name: success-200-isvc-3a16a.18c1eeaf6d957626 namespace: kserve-ci-e2e-test resourceVersion: "16786" uid: ff04da66-8856-4a89-89e4-56bdb04a6e29 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3a16a namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: c94d0f5b-eea8-4556-a991-ac47458d22ad kind: Event lastTimestamp: "2026-07-13T19:03:32Z" message: InferenceService [success-200-isvc-3a16a] is Ready metadata: creationTimestamp: "2026-07-13T19:03:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:32Z" name: success-200-isvc-3a16a.18c1eeba87fbd992 namespace: kserve-ci-e2e-test resourceVersion: "16782" uid: 215c4196-25ce-453b-b7d0-034279cdc010 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:22:44.736962Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc6d8e6714c namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 034e970a-c805-49a0-bf72-1bda8905b602 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4dc86-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc6e4725523 namespace: kserve-ci-e2e-test resourceVersion: "26279" uid: 4f4547bf-6d72-4630-8068-e6db3b0c48d6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc7169d58b6 namespace: kserve-ci-e2e-test resourceVersion: "26294" uid: c8c42331-10f0-41b6-a180-f5eeec7e00df reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc71802f989 namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: 45161179-daf3-4317-80e3-f3450e4cad6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc71c14710f namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: fd0bbc73-67b9-415e-84c7-58cb58296414 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc71cb0b6a5 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: ad4d6910-24a0-48e8-a5af-2e04d484e50f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc71cc9a094 namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: dd1f2a85-897b-4b31-82dd-e144d954efa6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc71ff9f0a9 namespace: kserve-ci-e2e-test resourceVersion: "26306" uid: 1f444441-d944-4718-8af7-4f5e608c540f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:22:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:45Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc72076938a namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 1b8c1a40-3751-4e19-b418-b7dddd704dd2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:22:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:23:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:32Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1efc723b6c38b namespace: kserve-ci-e2e-test resourceVersion: "26895" uid: ee97401d-389f-48a0-8722-42e783c0ec43 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:40:14Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:14Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1f0bb3bedfe30 namespace: kserve-ci-e2e-test resourceVersion: "34046" uid: a3c37814-b8b2-4085-b2b6-e18eee62f3fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:40:14Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:14Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1f0bb3bee7d1d namespace: kserve-ci-e2e-test resourceVersion: "34047" uid: 584ce089-f9b1-4844-ac76-c1a31b9b8c0d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: c0b288cb-227a-4c81-940b-a72e93e2405b kind: Event lastTimestamp: "2026-07-13T19:40:16Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:40:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:16Z" name: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j.18c1f0bbd8839108 namespace: kserve-ci-e2e-test resourceVersion: "34114" uid: e781df3e-0798-43c7-8c13-49d0f17e5fc8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4dc86-predictor-7db9988c5b namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: f6692e8b-0009-43da-adc2-48714a6834ef kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: 'Created pod: success-200-isvc-4dc86-predictor-7db9988c5b-ktd6j' metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-4dc86-predictor-7db9988c5b.18c1efc6d8539dd9 namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: 7b461d24-66e6-491c-9292-81238a791337 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 174f305c-0ed9-46dc-b202-edbe18e4f37f kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Scaled up replica set success-200-isvc-4dc86-predictor-7db9988c5b from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-4dc86-predictor.18c1efc6d604d19c namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 4800c116-78e7-44f7-a129-66108abbfc30 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: 1ca4935c-13e2-4003-ab5e-fdebcccd497b kind: Event lastTimestamp: "2026-07-13T19:23:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:14Z" name: success-200-isvc-4dc86-predictor.18c1efca54758870 namespace: kserve-ci-e2e-test resourceVersion: "26582" uid: a5a29f22-89fb-4261-a3b1-fa9e19494684 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:59Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: 1ca4935c-13e2-4003-ab5e-fdebcccd497b kind: Event lastTimestamp: "2026-07-13T19:23:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:14Z" name: success-200-isvc-4dc86-predictor.18c1efca54e4cde0 namespace: kserve-ci-e2e-test resourceVersion: "26583" uid: 5b6c88b8-229b-441b-9c94-4e628fb60830 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 1ca4935c-13e2-4003-ab5e-fdebcccd497b kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: success-200-isvc-4dc86-predictor.18c1efd152ce07ad namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: 9b9ebd42-72ec-4795-8ae3-f06037f921a4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4dc86-predictor namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 1ca4935c-13e2-4003-ab5e-fdebcccd497b kind: Event lastTimestamp: "2026-07-13T19:23:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:59Z" name: success-200-isvc-4dc86-predictor.18c1efd15336e10d namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: af3c4447-d159-4a81-b920-765d970f4eae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: ff1c65d9-0f9f-4333-ae06-ec4ab0f5c958 kind: Event lastTimestamp: "2026-07-13T19:23:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:42Z" name: success-200-isvc-4dc86.18c1efc6d61b8c91 namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: d1d446fe-086c-47bb-8198-3162bba15dfd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "26998" uid: ff1c65d9-0f9f-4333-ae06-ec4ab0f5c958 kind: Event lastTimestamp: "2026-07-13T19:23:42Z" message: InferenceService [success-200-isvc-4dc86] is Ready metadata: creationTimestamp: "2026-07-13T19:23:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:42Z" name: success-200-isvc-4dc86.18c1efd433a2c1a5 namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: 75560ff0-7dbb-45c4-b5d7-0fef6fe027ba reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:23.674560Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-68f71-predictor-dc77967f4-s6d9d to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef28466af94b namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: b194e902-4301-4fd8-8c34-ded5f674b792 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-68f71-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef284e5f7cc9 namespace: kserve-ci-e2e-test resourceVersion: "20150" uid: 4d413785-9f2a-4fc6-b81b-fb279309e08b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20122" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef28846b22e9 namespace: kserve-ci-e2e-test resourceVersion: "20177" uid: 984b41d7-2e73-46d1-a5be-2a527f9e0745 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef2885eced54 namespace: kserve-ci-e2e-test resourceVersion: "20179" uid: c925f7c9-7304-47f0-90d6-ec31a8ab96dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef2888f502ee namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: a8512494-3802-440d-a8f5-f1e78986d52d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef2889686923 namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: bce3cf01-6743-4883-a41a-a05a6143fa5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef288972de99 namespace: kserve-ci-e2e-test resourceVersion: "20182" uid: dd513eda-e2cf-4b2d-b7e6-2b9c242ed854 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef288cd1a906 namespace: kserve-ci-e2e-test resourceVersion: "20183" uid: 69e3271f-8abd-4342-aad3-ab492d46af9e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:11:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:24Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef288d490885 namespace: kserve-ci-e2e-test resourceVersion: "20184" uid: 7d0c06bc-dedd-4c5b-bbf1-d5dd8bf4a282 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:12:12Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:12Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef28df773853 namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: 8e64a699-954b-4541-822c-6888a17f968c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef3af1a013d1 namespace: kserve-ci-e2e-test resourceVersion: "21064" uid: 09471955-620e-47b9-8b96-4470bdf451cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef3af1a20feb namespace: kserve-ci-e2e-test resourceVersion: "21065" uid: 9a695c8f-cf1d-4423-81bb-e4c77157b2ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d namespace: kserve-ci-e2e-test resourceVersion: "20107" uid: 8692dd78-4ecc-4590-80b2-f69841a4e023 kind: Event lastTimestamp: "2026-07-13T19:12:47Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:47Z" name: success-200-isvc-68f71-predictor-dc77967f4-s6d9d.18c1ef3bbbc11683 namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: 192d245a-c2be-4905-be51-d140c2a27f7c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-68f71-predictor-dc77967f4 namespace: kserve-ci-e2e-test resourceVersion: "20086" uid: 5e85214c-1881-4a4f-9a44-3cb97840726d kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'Created pod: success-200-isvc-68f71-predictor-dc77967f4-s6d9d' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71-predictor-dc77967f4.18c1ef2845f42904 namespace: kserve-ci-e2e-test resourceVersion: "20106" uid: 018a30aa-9572-436e-a147-3640884a7a0d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20081" uid: 20b85030-357f-4e17-b6ca-45bfa704b2d3 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Scaled up replica set success-200-isvc-68f71-predictor-dc77967f4 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71-predictor.18c1ef28414a48b5 namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: f842ca4f-6473-4497-83ff-301414d95300 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: b6d3ea8d-547a-49db-843d-08d5a43d22a9 kind: Event lastTimestamp: "2026-07-13T19:11:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:38Z" name: success-200-isvc-68f71-predictor.18c1ef2bc0c041cd namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: bab47035-d3dc-448b-82d9-b981ed670562 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20083" uid: b6d3ea8d-547a-49db-843d-08d5a43d22a9 kind: Event lastTimestamp: "2026-07-13T19:11:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:11:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:38Z" name: success-200-isvc-68f71-predictor.18c1ef2bc1877206 namespace: kserve-ci-e2e-test resourceVersion: "20311" uid: c9c8cac3-bd60-4b6c-b9d5-bf0134d70b70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:11:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: b6d3ea8d-547a-49db-843d-08d5a43d22a9 kind: Event lastTimestamp: "2026-07-13T19:12:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:38Z" name: success-200-isvc-68f71-predictor.18c1ef2f40678a36 namespace: kserve-ci-e2e-test resourceVersion: "20986" uid: 84ab2c5a-7cc0-4db2-80c4-e17a87080775 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:11:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-68f71-predictor namespace: kserve-ci-e2e-test resourceVersion: "20309" uid: b6d3ea8d-547a-49db-843d-08d5a43d22a9 kind: Event lastTimestamp: "2026-07-13T19:12:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:11:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:38Z" name: success-200-isvc-68f71-predictor.18c1ef2f40db5537 namespace: kserve-ci-e2e-test resourceVersion: "20987" uid: a85c679d-14f5-4b3f-b010-189a1e6fac91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: e599764b-34cf-40da-a4f5-eef8f35ecb1a kind: Event lastTimestamp: "2026-07-13T19:12:22Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:22Z" name: success-200-isvc-68f71.18c1ef284140688e namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: 825a69eb-aa5e-4eac-82d7-a9ae3863e14d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: e599764b-34cf-40da-a4f5-eef8f35ecb1a kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'Failed to update status for InferenceService "success-200-isvc-68f71": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68f71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71.18c1ef28421708c5 namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: 92f30c19-e0fd-410f-94f8-af2ce03e08d4 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20068" uid: e599764b-34cf-40da-a4f5-eef8f35ecb1a kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-68f71": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-68f71.18c1ef2842175f96 namespace: kserve-ci-e2e-test resourceVersion: "20096" uid: 9af7714f-c5cf-4239-a5e7-5d848e0982b6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-68f71 namespace: kserve-ci-e2e-test resourceVersion: "20819" uid: e599764b-34cf-40da-a4f5-eef8f35ecb1a kind: Event lastTimestamp: "2026-07-13T19:12:22Z" message: InferenceService [success-200-isvc-68f71] is Ready metadata: creationTimestamp: "2026-07-13T19:12:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:22Z" name: success-200-isvc-68f71.18c1ef35ef610335 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 085ebdba-8bca-41b3-8b81-6b1ba7a2a9a6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:01:58.717176Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4bc55579c namespace: kserve-ci-e2e-test resourceVersion: "15658" uid: d4a80a2f-02cf-4b9c-b72f-6da6b3b3e281 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Add eth0 [10.134.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4d639447b namespace: kserve-ci-e2e-test resourceVersion: "15712" uid: 2f5fdec0-ac85-434a-b9b7-faba74e53803 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4d790cc8d namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: 45da6b9a-296a-491a-a7a9-2984f8330c2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4db3ba8c2 namespace: kserve-ci-e2e-test resourceVersion: "15715" uid: 5b9a2101-118e-453e-9e62-2ed1e2006b5f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4dbcdddd1 namespace: kserve-ci-e2e-test resourceVersion: "15716" uid: 1c3727f5-cbc2-4dbb-9fbd-ade95c6fbdf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4dbd90275 namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: 01314093-6ed9-46f2-9ed7-9211091e5e9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4de7825b1 namespace: kserve-ci-e2e-test resourceVersion: "15718" uid: 53932d70-b54b-41f2-85f5-737241324f72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:01:59Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:01:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:59Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea4deed2255 namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: aae753f7-efb9-4c42-9cb5-c18bf704c36d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:11:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:26Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1eea509dba103 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: 24cb2b66-c9f8-492e-9e01-afc355fbbce2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1ef2842a488bc namespace: kserve-ci-e2e-test resourceVersion: "20098" uid: 9910ef59-345c-4daf-8eb4-e21e8722775e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1ef2842a5a5a1 namespace: kserve-ci-e2e-test resourceVersion: "20099" uid: d5275274-5869-4071-b2ba-1f67b1c860e7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: e2ec0e05-4658-425d-87c8-379dbcd0769d kind: Event lastTimestamp: "2026-07-13T19:11:26Z" message: 'Readiness probe failed: Get "https://10.134.0.24:8643/healthz": dial tcp 10.134.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:26Z" name: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4.18c1ef28d29cfb28 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: b11a3d3c-ad69-452f-949b-a1faa0c5f54b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-91c3d-predictor-cc8f58fd namespace: kserve-ci-e2e-test resourceVersion: "15634" uid: 06c9c9a6-f56e-4298-b013-f42773bf2608 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'Created pod: success-200-isvc-91c3d-predictor-cc8f58fd-kbsz4' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-91c3d-predictor-cc8f58fd.18c1eea4bbade360 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 17667dff-5f20-40e1-ab63-b9d8dd17ba05 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 4a662328-c438-458a-9954-721ff6345127 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Scaled up replica set success-200-isvc-91c3d-predictor-cc8f58fd from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-91c3d-predictor.18c1eea4b840d12f namespace: kserve-ci-e2e-test resourceVersion: "15635" uid: 27a38160-801e-4fad-8486-ce27568b57fe reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 60a8ca56-3d1f-4333-8203-0a2c9164fe27 kind: Event lastTimestamp: "2026-07-13T19:02:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:28Z" name: success-200-isvc-91c3d-predictor.18c1eea836f2a928 namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: bb53a192-a830-48a8-8653-aa710733c1f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:02:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15633" uid: 60a8ca56-3d1f-4333-8203-0a2c9164fe27 kind: Event lastTimestamp: "2026-07-13T19:02:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:02:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:02:28Z" name: success-200-isvc-91c3d-predictor.18c1eea8376051da namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: 402d84c0-c4ac-4045-a0c4-ec1247350e46 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 60a8ca56-3d1f-4333-8203-0a2c9164fe27 kind: Event lastTimestamp: "2026-07-13T19:03:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:13Z" name: success-200-isvc-91c3d-predictor.18c1eeaf34d5ccc4 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 16df160d-cf84-491b-a9df-b12874715eb2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:02:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-91c3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "15872" uid: 60a8ca56-3d1f-4333-8203-0a2c9164fe27 kind: Event lastTimestamp: "2026-07-13T19:03:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:02:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:13Z" name: success-200-isvc-91c3d-predictor.18c1eeaf353ef6bb namespace: kserve-ci-e2e-test resourceVersion: "16614" uid: 1d92b452-071e-4191-bcfd-c714aa058a0a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 0e091d8b-fb0d-456c-bd70-08748c626cd4 kind: Event lastTimestamp: "2026-07-13T19:02:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:56Z" name: success-200-isvc-91c3d.18c1eea4b8780e06 namespace: kserve-ci-e2e-test resourceVersion: "16443" uid: 31ea8c05-7be2-49db-9ced-235c7fddfa62 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 0e091d8b-fb0d-456c-bd70-08748c626cd4 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'Failed to update status for InferenceService "success-200-isvc-91c3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-91c3d.18c1eea4b8dac783 namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 521b4c6a-4f71-4fe0-bb17-62953704b478 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "15609" uid: 0e091d8b-fb0d-456c-bd70-08748c626cd4 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-91c3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:58Z" name: success-200-isvc-91c3d.18c1eea4b8db17c0 namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: effede84-26da-40b9-8570-02c7d9030d2f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:02:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16441" uid: 0e091d8b-fb0d-456c-bd70-08748c626cd4 kind: Event lastTimestamp: "2026-07-13T19:02:56Z" message: InferenceService [success-200-isvc-91c3d] is Ready metadata: creationTimestamp: "2026-07-13T19:02:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:02:56Z" name: success-200-isvc-91c3d.18c1eeb219eea306 namespace: kserve-ci-e2e-test resourceVersion: "16442" uid: fc44b9eb-26c2-4f06-a824-f7805bc029f8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:13:19.954651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef435940440b namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 962592c0-212c-4162-a1bc-acd7aebd742c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef437346d89b namespace: kserve-ci-e2e-test resourceVersion: "21643" uid: a2eedf40-9950-46fa-b4d6-c76dfe9e193c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef4374c68002 namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: 8a91fba8-a8e7-42c7-9e8e-73286a73070a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef43791008aa namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: 1cd1e846-4048-4126-a642-0ba12721edf7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef43798c2a3b namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: 4c6f198c-57a0-4ba0-ab18-d47923f1494b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef4379979f57 namespace: kserve-ci-e2e-test resourceVersion: "21651" uid: 583d0548-25ed-452c-a581-7f8211f99e76 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef437cc2e9c8 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: a817c4ef-a1d0-4899-920d-bc99ffd83735 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:13:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:13:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:20Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef437d433257 namespace: kserve-ci-e2e-test resourceVersion: "21656" uid: fd4807cc-d28a-46d9-8706-b6674a28e9c2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:13:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:22:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:47Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1ef43c552c263 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: 7aedc46f-2817-4958-9097-8105960dc74f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1efc6d53572c9 namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: ea6ac06f-6844-4092-adeb-834127cff463 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:22:44Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:44Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1efc6d536a049 namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: 34659e51-1fdc-4af4-9091-75a0627e543c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8 namespace: kserve-ci-e2e-test resourceVersion: "21596" uid: 25ec8725-5321-47b2-ab02-605a24f0fdca kind: Event lastTimestamp: "2026-07-13T19:22:47Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:47Z" name: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8.18c1efc78dbfee74 namespace: kserve-ci-e2e-test resourceVersion: "26341" uid: a7d96b0a-e959-4cf4-9959-e12ada9aae88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a68dc-predictor-654cb8897d namespace: kserve-ci-e2e-test resourceVersion: "21576" uid: cbb62a16-d030-47a1-9fe8-673cbc898940 kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: 'Created pod: success-200-isvc-a68dc-predictor-654cb8897d-4l8c8' metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-a68dc-predictor-654cb8897d.18c1ef4358939a00 namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 342084a3-0473-4c2e-9046-5357f27f6eb3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: c8b1055f-b37c-4ec5-8687-d11c4b654d39 kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Scaled up replica set success-200-isvc-a68dc-predictor-654cb8897d from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-a68dc-predictor.18c1ef435689e8c4 namespace: kserve-ci-e2e-test resourceVersion: "21579" uid: f40a822f-70d3-49ca-a8a4-a08208a51ec7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 9f6ca75d-67a2-46b8-bf36-71bc5e67d0d0 kind: Event lastTimestamp: "2026-07-13T19:14:04Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:04Z" name: success-200-isvc-a68dc-predictor.18c1ef46d5695d49 namespace: kserve-ci-e2e-test resourceVersion: "22079" uid: f38e45f5-67d8-4b56-8ac5-5cd0fe90709f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:13:34Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 9f6ca75d-67a2-46b8-bf36-71bc5e67d0d0 kind: Event lastTimestamp: "2026-07-13T19:14:04Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:13:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:04Z" name: success-200-isvc-a68dc-predictor.18c1ef46d5d00a7a namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: d7415900-ccaa-4e5a-9c8d-f7e8b89b5603 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 9f6ca75d-67a2-46b8-bf36-71bc5e67d0d0 kind: Event lastTimestamp: "2026-07-13T19:14:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:50Z" name: success-200-isvc-a68dc-predictor.18c1ef4a5473f137 namespace: kserve-ci-e2e-test resourceVersion: "22468" uid: 604fc01c-65f0-4470-9830-dd4213a393e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:49Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a68dc-predictor namespace: kserve-ci-e2e-test resourceVersion: "21789" uid: 9f6ca75d-67a2-46b8-bf36-71bc5e67d0d0 kind: Event lastTimestamp: "2026-07-13T19:14:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:50Z" name: success-200-isvc-a68dc-predictor.18c1ef4a54d54d52 namespace: kserve-ci-e2e-test resourceVersion: "22469" uid: bae8c3f3-aa08-493e-9911-c12fc0633d85 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 9acefe30-d8c3-4d36-8baf-49c33f6c3b8e kind: Event lastTimestamp: "2026-07-13T19:14:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:17Z" name: success-200-isvc-a68dc.18c1ef4356b76c65 namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 0cfb53ae-d21a-45da-8c77-a854e1a9dca5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 9acefe30-d8c3-4d36-8baf-49c33f6c3b8e kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: 'Failed to update status for InferenceService "success-200-isvc-a68dc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a68dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-a68dc.18c1ef435778431f namespace: kserve-ci-e2e-test resourceVersion: "21588" uid: b98ad3d0-e316-4818-a48e-682483bba484 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: 9acefe30-d8c3-4d36-8baf-49c33f6c3b8e kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a68dc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-a68dc.18c1ef4357788b02 namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: e3fc0c10-1208-4134-8d23-604a9f9c4ab3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:14:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a68dc namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: 9acefe30-d8c3-4d36-8baf-49c33f6c3b8e kind: Event lastTimestamp: "2026-07-13T19:14:17Z" message: InferenceService [success-200-isvc-a68dc] is Ready metadata: creationTimestamp: "2026-07-13T19:14:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:14:17Z" name: success-200-isvc-a68dc.18c1ef50d4d9c99e namespace: kserve-ci-e2e-test resourceVersion: "22195" uid: 7dfabf3a-4a89-4dbf-b6ac-cd6c340f6bb8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:22:08.908696Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25802" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbe815e818c namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: dc34dccd-2e14-45c2-b8f9-382156607929 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-b5afb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:09Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbe8eb02260 namespace: kserve-ci-e2e-test resourceVersion: "25857" uid: 0a7802f4-c5f7-40d3-a974-8897a646ff62 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25820" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:22:09Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbebf4b91ad namespace: kserve-ci-e2e-test resourceVersion: "25872" uid: 087b34c0-92e9-4558-bd63-beb938fb2b6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:09Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbec0aace18 namespace: kserve-ci-e2e-test resourceVersion: "25874" uid: 4d9d8568-61e8-43d2-9b38-552ffad8a95a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbec5aa02f9 namespace: kserve-ci-e2e-test resourceVersion: "25877" uid: c62ef526-ffb3-4165-9cf7-6de3653bab16 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbec62fb5bc namespace: kserve-ci-e2e-test resourceVersion: "25878" uid: a597de70-0052-4b34-adc0-60670f077209 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbec63c2542 namespace: kserve-ci-e2e-test resourceVersion: "25879" uid: 434d414c-3563-4c73-8f26-251781dcf796 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbec9dbc4c2 namespace: kserve-ci-e2e-test resourceVersion: "25882" uid: a3f2a25d-6bb3-4c0b-aec7-3622a93c5f3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:10Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:10Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbeca59176c namespace: kserve-ci-e2e-test resourceVersion: "25884" uid: a6072ad3-e98d-41c5-9c82-bf596738f24f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:22:57Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:57Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efbf3049be05 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: b769d89d-255d-4406-a2bd-729cb66e8da0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efd13082d78f namespace: kserve-ci-e2e-test resourceVersion: "26764" uid: 2a706c1c-178c-46fc-8f29-0a5c30ce6d9e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:23:29Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:29Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efd130835c76 namespace: kserve-ci-e2e-test resourceVersion: "26766" uid: 4abf220b-f18a-4b74-81b0-acaf6be4d04c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4 namespace: kserve-ci-e2e-test resourceVersion: "25804" uid: 893f2442-98f0-45b6-bccd-17a4f7826dbb kind: Event lastTimestamp: "2026-07-13T19:23:32Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:23:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:32Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4.18c1efd20cc75781 namespace: kserve-ci-e2e-test resourceVersion: "26902" uid: 720318b4-efe9-4154-9a26-7387210c9b5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-b5afb-predictor-5d6dcc6dc4 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: e017c1bc-d627-4683-b70c-ee0f205cdced kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'Created pod: success-200-isvc-b5afb-predictor-5d6dcc6dc4-wgkz4' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-b5afb-predictor-5d6dcc6dc4.18c1efbe80f028f6 namespace: kserve-ci-e2e-test resourceVersion: "25803" uid: b8963e1a-de4d-426a-90c3-8d073bfb4f28 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: f06f56f0-cc1b-4c66-9ac7-2eccbbf26f99 kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: Scaled up replica set success-200-isvc-b5afb-predictor-5d6dcc6dc4 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-b5afb-predictor.18c1efbe7d6e666f namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 72bac81d-fd82-4c2b-8515-59f5765e514f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 60acbeaf-ecde-477b-b1b0-5fbfa2beaf27 kind: Event lastTimestamp: "2026-07-13T19:22:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:38Z" name: success-200-isvc-b5afb-predictor.18c1efc1fc1409d5 namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: ae14c8e7-87f3-4595-b36e-a6b23038ae17 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:22:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 60acbeaf-ecde-477b-b1b0-5fbfa2beaf27 kind: Event lastTimestamp: "2026-07-13T19:22:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:22:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:22:38Z" name: success-200-isvc-b5afb-predictor.18c1efc1fc7f21b7 namespace: kserve-ci-e2e-test resourceVersion: "26130" uid: 5da19d52-84c5-4ad6-a918-79936a9c1e4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:22:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 60acbeaf-ecde-477b-b1b0-5fbfa2beaf27 kind: Event lastTimestamp: "2026-07-13T19:23:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:23Z" name: success-200-isvc-b5afb-predictor.18c1efc8fa4a48be namespace: kserve-ci-e2e-test resourceVersion: "26687" uid: 7baa11c8-f4a9-41bc-9f1d-a9dcccbeb693 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-13T19:22:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-b5afb-predictor namespace: kserve-ci-e2e-test resourceVersion: "26012" uid: 60acbeaf-ecde-477b-b1b0-5fbfa2beaf27 kind: Event lastTimestamp: "2026-07-13T19:23:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:22:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:23Z" name: success-200-isvc-b5afb-predictor.18c1efc8faa1b469 namespace: kserve-ci-e2e-test resourceVersion: "26688" uid: dc55a550-4821-4090-98df-c2539a30c890 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: a79acbc4-65b9-48ad-8f44-684da1364925 kind: Event lastTimestamp: "2026-07-13T19:23:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:07Z" name: success-200-isvc-b5afb.18c1efbe7d976e9f namespace: kserve-ci-e2e-test resourceVersion: "26524" uid: 3010af14-9ea1-40bf-8b43-70cd6ae2879c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: a79acbc4-65b9-48ad-8f44-684da1364925 kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'Failed to update status for InferenceService "success-200-isvc-b5afb": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b5afb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-b5afb.18c1efbe7e0ec285 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: eb2b1740-b9a0-4788-b1bc-48f9d07c11f2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "25769" uid: a79acbc4-65b9-48ad-8f44-684da1364925 kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-b5afb": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-b5afb.18c1efbe7e0f26c3 namespace: kserve-ci-e2e-test resourceVersion: "25799" uid: 7c9937e0-60e6-4c80-a847-0a09fea985fb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-b5afb namespace: kserve-ci-e2e-test resourceVersion: "26522" uid: a79acbc4-65b9-48ad-8f44-684da1364925 kind: Event lastTimestamp: "2026-07-13T19:23:07Z" message: InferenceService [success-200-isvc-b5afb] is Ready metadata: creationTimestamp: "2026-07-13T19:23:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:07Z" name: success-200-isvc-b5afb.18c1efcc402bfc5f namespace: kserve-ci-e2e-test resourceVersion: "26523" uid: 48312a9f-635b-4746-8d7b-8159a9ad09d2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:11:59.752488Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20539" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e56dd-predictor-6b959756d6-cslzx to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30acd45faf namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 89bcc9df-0758-4400-91ec-c3c2432e4749 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e56dd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30bbba810e namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: e1952b41-169b-49f8-9206-753e2e439220 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20548" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30ea8e36e0 namespace: kserve-ci-e2e-test resourceVersion: "20603" uid: 139f633b-1951-4b3d-b516-cdc3ce9017eb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30ecf8d329 namespace: kserve-ci-e2e-test resourceVersion: "20605" uid: 16c372c0-5851-4396-8cb9-c1f75ae42d50 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30effccbb4 namespace: kserve-ci-e2e-test resourceVersion: "20606" uid: baf47605-b410-4d53-8706-ef191fa0c0b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30f072bc00 namespace: kserve-ci-e2e-test resourceVersion: "20607" uid: b2387cee-03e6-4615-b23c-a70b20dbb5f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30f07c9e97 namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: 60fd6c57-ae24-44f1-ada8-42697190df7e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30f314d0bc namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 0ed8467e-91a0-4feb-95e8-055763c04853 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:00Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:00Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef30f38a1300 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: bd89c780-5224-48ca-ac0e-43332b95179d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-13T19:12:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:12:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:47Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef310ec4606c namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: bffd95df-c537-4448-8a56-6003d698d81e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef435431047c namespace: kserve-ci-e2e-test resourceVersion: "21559" uid: b7198361-b6a6-4089-ae6c-7ab1b10f3842 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:13:19Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:13:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:19Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef435431c214 namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: dba457d1-a852-4f08-980d-cb9044145af1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx namespace: kserve-ci-e2e-test resourceVersion: "20541" uid: 686d93a4-2a33-4158-be50-5531c78df11f kind: Event lastTimestamp: "2026-07-13T19:13:22Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:13:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:13:22Z" name: success-200-isvc-e56dd-predictor-6b959756d6-cslzx.18c1ef43eb0652d4 namespace: kserve-ci-e2e-test resourceVersion: "21685" uid: 5e3aadf9-9b52-4e6e-8380-cfb061f8f7c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e56dd-predictor-6b959756d6 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: c49e341a-6ccc-4e4e-9278-33b66c45168d kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'Created pod: success-200-isvc-e56dd-predictor-6b959756d6-cslzx' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-e56dd-predictor-6b959756d6.18c1ef30ac0be5fd namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 4447dc39-efcb-4ee6-9557-9e4419b315d8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20519" uid: 8597f3df-8bb3-4151-9620-9a8355276773 kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: Scaled up replica set success-200-isvc-e56dd-predictor-6b959756d6 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-e56dd-predictor.18c1ef30a7428a45 namespace: kserve-ci-e2e-test resourceVersion: "20529" uid: 8d9ce7ba-cbca-449b-931d-c7c4147b3596 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: df56e591-0be0-4d01-b98c-1c0d6ee57267 kind: Event lastTimestamp: "2026-07-13T19:12:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:14Z" name: success-200-isvc-e56dd-predictor.18c1ef3425c9b41f namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: 6e9d6f06-85a9-430c-a1d4-e002424ea40d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20525" uid: df56e591-0be0-4d01-b98c-1c0d6ee57267 kind: Event lastTimestamp: "2026-07-13T19:12:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:14Z" name: success-200-isvc-e56dd-predictor.18c1ef342657df14 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: d48fb07b-3c1c-4f8d-ae6f-8d2301166112 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:12:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: df56e591-0be0-4d01-b98c-1c0d6ee57267 kind: Event lastTimestamp: "2026-07-13T19:13:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:14Z" name: success-200-isvc-e56dd-predictor.18c1ef37a5105684 namespace: kserve-ci-e2e-test resourceVersion: "21479" uid: 37738262-22d6-42ab-839f-bead446661d4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:12:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e56dd-predictor namespace: kserve-ci-e2e-test resourceVersion: "20744" uid: df56e591-0be0-4d01-b98c-1c0d6ee57267 kind: Event lastTimestamp: "2026-07-13T19:13:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:14Z" name: success-200-isvc-e56dd-predictor.18c1ef37a57e1d72 namespace: kserve-ci-e2e-test resourceVersion: "21480" uid: 2f7d4e88-8ed6-4b3d-97b2-1e5ea66965e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 5362c4fb-334d-43b7-9c52-b977b0c790cf kind: Event lastTimestamp: "2026-07-13T19:12:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:57Z" name: success-200-isvc-e56dd.18c1ef30a79978b8 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: b30bb753-c6c8-4d6b-8b8a-c348eea963dd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 5362c4fb-334d-43b7-9c52-b977b0c790cf kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'Failed to update status for InferenceService "success-200-isvc-e56dd": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e56dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-e56dd.18c1ef30a8e49278 namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: d64db8da-9e7b-4526-8a68-0d1cf79420c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:59Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: 5362c4fb-334d-43b7-9c52-b977b0c790cf kind: Event lastTimestamp: "2026-07-13T19:11:59Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e56dd": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:11:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:11:59Z" name: success-200-isvc-e56dd.18c1ef30a8e4f79d namespace: kserve-ci-e2e-test resourceVersion: "20536" uid: d658b624-aedd-4ae8-8ac7-521b6ef2f597 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e56dd namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 5362c4fb-334d-43b7-9c52-b977b0c790cf kind: Event lastTimestamp: "2026-07-13T19:12:57Z" message: InferenceService [success-200-isvc-e56dd] is Ready metadata: creationTimestamp: "2026-07-13T19:12:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:57Z" name: success-200-isvc-e56dd.18c1ef3e1e68cdf6 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: 221e74b5-655b-4f33-8267-f4acca4aa8ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:12:43.933539Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21098" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b to ip-10-0-139-100.ec2.internal metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3af639bf4a namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 5bb78f1e-86fa-4275-aa8c-64b50ba856fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ef1db-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:44Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3afd6bffcb namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: 490f05fe-2dda-4358-af99-43902cd6282c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:44Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:12:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:12:44Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b3486102e namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 0a3cde6b-e1d2-4311-bfc0-a78e101d5678 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1730" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b36275654 namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: ccb2b460-ee37-4f2c-8f3a-f43743fde768 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b39207cdd namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: b5b86020-6a5f-4f5e-b48b-3635f4e4542d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b39be8f03 namespace: kserve-ci-e2e-test resourceVersion: "21162" uid: b10352ab-2e12-4a0f-80d9-e38d1f972264 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b39ca54b2 namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 238e1d8c-208a-44c4-b8d8-15cd25c1f6d0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b3d11bf6d namespace: kserve-ci-e2e-test resourceVersion: "21168" uid: 882baba8-ac5f-4aa4-a9f6-0f490776515a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:12:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:12:45Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b3daa82db namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: c18a41b7-8830-405d-a1c0-c386fea05dad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-13T19:12:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:22:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:12:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:11Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1ef3b59278c3a namespace: kserve-ci-e2e-test resourceVersion: "25909" uid: aacffbec-a7c0-42f1-a9d6-e49211b2da2a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1efbe7def0fae namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: 60e0647d-6620-4300-8341-959d491555ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:22:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-13T19:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:08Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1efbe7def7851 namespace: kserve-ci-e2e-test resourceVersion: "25797" uid: 6454f834-8a76-4a0b-93da-4e508ff0472a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:22:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b namespace: kserve-ci-e2e-test resourceVersion: "21099" uid: 9217e6f3-d8a9-4158-83ce-68156f4f0caa kind: Event lastTimestamp: "2026-07-13T19:22:11Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-13T19:22:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:22:11Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b.18c1efbf21c08eee namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: c7b58c79-7cd5-468c-b74b-8fcf47949202 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-139-100.ec2.internal source: component: kubelet host: ip-10-0-139-100.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ef1db-predictor-5bdcf6bfc4 namespace: kserve-ci-e2e-test resourceVersion: "21087" uid: 4b54a255-f195-4298-bdd4-67cce6ec2711 kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: 'Created pod: success-200-isvc-ef1db-predictor-5bdcf6bfc4-v2s9b' metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-ef1db-predictor-5bdcf6bfc4.18c1ef3af5e99e13 namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 8b2e0e98-314b-4a3c-a6c3-0f70fd4dc70a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: fb0792ee-5236-4862-9428-6e8c69fccc04 kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: Scaled up replica set success-200-isvc-ef1db-predictor-5bdcf6bfc4 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-ef1db-predictor.18c1ef3af43cdda1 namespace: kserve-ci-e2e-test resourceVersion: "21089" uid: b2c13a7d-b60c-47be-840a-93ffe93b2cbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: ae569179-4ad5-462a-a536-5863de646617 kind: Event lastTimestamp: "2026-07-13T19:13:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:13Z" name: success-200-isvc-ef1db-predictor.18c1ef3e73018a5a namespace: kserve-ci-e2e-test resourceVersion: "21466" uid: 82c6b17f-4813-4bad-96d9-21084f660ad0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:12:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21086" uid: ae569179-4ad5-462a-a536-5863de646617 kind: Event lastTimestamp: "2026-07-13T19:13:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:12:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:13:13Z" name: success-200-isvc-ef1db-predictor.18c1ef3e736a30ca namespace: kserve-ci-e2e-test resourceVersion: "21467" uid: c3d130ea-6b77-4514-9b15-4bf164fc47b6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: ae569179-4ad5-462a-a536-5863de646617 kind: Event lastTimestamp: "2026-07-13T19:14:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:13Z" name: success-200-isvc-ef1db-predictor.18c1ef45716fdaab namespace: kserve-ci-e2e-test resourceVersion: "22163" uid: 7acc1a7a-b229-4db6-a200-2f412c3ce414 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-13T19:13:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ef1db-predictor namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: ae569179-4ad5-462a-a536-5863de646617 kind: Event lastTimestamp: "2026-07-13T19:14:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:13:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:14:13Z" name: success-200-isvc-ef1db-predictor.18c1ef4571bdc820 namespace: kserve-ci-e2e-test resourceVersion: "22164" uid: a815f232-b8f1-4adc-9e38-414e86a840a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: b39043a3-0161-4880-a548-212b4e63274e kind: Event lastTimestamp: "2026-07-13T19:13:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:41Z" name: success-200-isvc-ef1db.18c1ef3af4527fbc namespace: kserve-ci-e2e-test resourceVersion: "21862" uid: c83e36e3-ef3e-4402-ae62-5b511c4575f4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: b39043a3-0161-4880-a548-212b4e63274e kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: 'Failed to update status for InferenceService "success-200-isvc-ef1db": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ef1db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-ef1db.18c1ef3af4c78d4b namespace: kserve-ci-e2e-test resourceVersion: "21095" uid: c8614f3e-e3dc-49eb-aa94-95eaaeb976ea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:12:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21061" uid: b39043a3-0161-4880-a548-212b4e63274e kind: Event lastTimestamp: "2026-07-13T19:12:43Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ef1db": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-13T19:12:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:12:43Z" name: success-200-isvc-ef1db.18c1ef3af4c7e063 namespace: kserve-ci-e2e-test resourceVersion: "21096" uid: 91179f51-c628-43ed-8219-d3de7c6e8a21 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:13:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ef1db namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: b39043a3-0161-4880-a548-212b4e63274e kind: Event lastTimestamp: "2026-07-13T19:13:41Z" message: InferenceService [success-200-isvc-ef1db] is Ready metadata: creationTimestamp: "2026-07-13T19:13:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:13:41Z" name: success-200-isvc-ef1db.18c1ef486b9b6cb7 namespace: kserve-ci-e2e-test resourceVersion: "21860" uid: bee5453a-b1ba-48f4-b18d-31ae6c1b833d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:01:44.348092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-19366-84654b9d67-xpxcb to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:01:44Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:01:44Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea163de8683 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: c600a725-a7a2-4f31-890e-765e5d07d871 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:01:44Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:45Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-19366-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:45Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea174675437 namespace: kserve-ci-e2e-test resourceVersion: "15462" uid: 42a1af7c-f1a8-4977-b88a-5ffd24d83110 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:46Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:01:46Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea1d6a029c2 namespace: kserve-ci-e2e-test resourceVersion: "15473" uid: 607074d9-5d22-410e-8e32-d174e144648f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:46Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" metadata: creationTimestamp: "2026-07-13T19:01:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:46Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea1d7d47ced namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 6e27bca1-af14-4ea6-8321-db54f6b62efd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:48Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" in 2.525s (2.525s including waiting). Image size: 218072665 bytes.' metadata: creationTimestamp: "2026-07-13T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:48Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea26e609315 namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 7f111542-1963-4dd7-ab19-234379dc9052 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:48Z" message: 'Created container: switch-graph-19366' metadata: creationTimestamp: "2026-07-13T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:48Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea27132acfe namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: c72355f0-481f-410e-a933-cd9b20cbd92e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:48Z" message: Started container switch-graph-19366 metadata: creationTimestamp: "2026-07-13T19:01:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:48Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea2718f7d4c namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 346edd98-06d5-4f8c-98d1-63c295fec4c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:01:58Z" message: Stopping container switch-graph-19366 metadata: creationTimestamp: "2026-07-13T19:01:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:01:58Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea4ae4e2189 namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 34fb0fc6-3f77-42a2-af48-5ffe1e5b2133 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:02:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-19366} kind: Pod name: switch-graph-19366-84654b9d67-xpxcb namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 8c9c69cd-3892-4eb5-bb2c-649d861d9468 kind: Event lastTimestamp: "2026-07-13T19:02:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:02:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:02:25Z" name: switch-graph-19366-84654b9d67-xpxcb.18c1eea5135ae470 namespace: kserve-ci-e2e-test resourceVersion: "15974" uid: 8adf16e8-5aba-466b-a8f6-1c51a54283bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:44Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-19366-84654b9d67 namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 628fbec4-a00b-42e2-ae14-6a49c80bb3c9 kind: Event lastTimestamp: "2026-07-13T19:01:44Z" message: 'Created pod: switch-graph-19366-84654b9d67-xpxcb' metadata: creationTimestamp: "2026-07-13T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:44Z" name: switch-graph-19366-84654b9d67.18c1eea163703246 namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: 40549203-7852-417c-bc61-c538e8dd2a22 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:44Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-19366 namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: 9ffdcb4e-c60a-4c09-8ee5-3594eb475213 kind: Event lastTimestamp: "2026-07-13T19:01:44Z" message: Scaled up replica set switch-graph-19366-84654b9d67 from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:01:44Z" name: switch-graph-19366.18c1eea1627ce85f namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: 5e4f8140-6e22-4cfa-8efc-df895565a193 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:01:44Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-19366 namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 4a2e23d7-84c3-4b8f-a80c-ae760ab51168 kind: Event lastTimestamp: "2026-07-13T19:01:44Z" message: InferenceGraph [switch-graph-19366] is Ready metadata: creationTimestamp: "2026-07-13T19:01:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:01:44Z" name: switch-graph-19366.18c1eea16838d282 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: ce27abcb-771f-422a-9b3c-196aeec0071c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:23:54.802097Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27115" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-4dc86-84fd9855bd-pgkxz to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:23:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:23:54Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd7291b9379 namespace: kserve-ci-e2e-test resourceVersion: "27120" uid: b17d9849-30d6-4e4c-a795-02317b98f0ff reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-4dc86-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:55Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd73782a47c namespace: kserve-ci-e2e-test resourceVersion: "27130" uid: e1d2817f-1de8-4e13-99aa-7627d5a131de reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:55Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:23:55Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd766241d65 namespace: kserve-ci-e2e-test resourceVersion: "27138" uid: cca23bf5-53c2-4429-b26f-fef4d4fb84ac reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dc86} kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:55Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd7676917f3 namespace: kserve-ci-e2e-test resourceVersion: "27140" uid: bfd8dd2e-7cae-4948-94b6-9a4ecc9f7fd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dc86} kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: 'Created container: switch-graph-4dc86' metadata: creationTimestamp: "2026-07-13T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:55Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd76a490189 namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: 4056e673-417c-41d8-8907-0b22e4881b8d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dc86} kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:23:55Z" message: Started container switch-graph-4dc86 metadata: creationTimestamp: "2026-07-13T19:23:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:23:55Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1efd76ab95537 namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 6d12d7c1-8fc3-46ff-ad25-e0c317db32a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:40:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dc86} kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:40:14Z" message: Stopping container switch-graph-4dc86 metadata: creationTimestamp: "2026-07-13T19:40:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:14Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1f0bb33c8512b namespace: kserve-ci-e2e-test resourceVersion: "34028" uid: 726414b4-1331-469e-ba29-434191afd76f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-13T19:40:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-4dc86} kind: Pod name: switch-graph-4dc86-84fd9855bd-pgkxz namespace: kserve-ci-e2e-test resourceVersion: "27118" uid: bf1ff38f-cd03-4bff-9c98-884d594241ae kind: Event lastTimestamp: "2026-07-13T19:40:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:40:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:40:37Z" name: switch-graph-4dc86-84fd9855bd-pgkxz.18c1f0bc074ede3f namespace: kserve-ci-e2e-test resourceVersion: "34274" uid: ab58a4a9-df7b-4acd-bff1-290dc1ed1ff9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-4dc86-84fd9855bd namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: 261eb5c9-f208-4d8e-96df-79f11fe3e7d7 kind: Event lastTimestamp: "2026-07-13T19:23:54Z" message: 'Created pod: switch-graph-4dc86-84fd9855bd-pgkxz' metadata: creationTimestamp: "2026-07-13T19:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:54Z" name: switch-graph-4dc86-84fd9855bd.18c1efd728b3368b namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: c2558e3a-83f8-4cd7-b558-295923e7cab9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "27110" uid: 2bff405f-345b-49c2-8a44-39d2e5c4814c kind: Event lastTimestamp: "2026-07-13T19:23:54Z" message: Scaled up replica set switch-graph-4dc86-84fd9855bd from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:23:54Z" name: switch-graph-4dc86.18c1efd7279fe044 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 7cb9d711-78ed-43b8-a44b-4064082e8758 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:23:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "27126" uid: e6f6f0c3-7f79-4228-a59b-001b777e87e3 kind: Event lastTimestamp: "2026-07-13T19:23:54Z" message: InferenceGraph [switch-graph-4dc86] is Ready metadata: creationTimestamp: "2026-07-13T19:23:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:23:54Z" name: switch-graph-4dc86.18c1efd72e266301 namespace: kserve-ci-e2e-test resourceVersion: "27128" uid: 2d5b9a66-26e6-4d1d-b17d-837a11059ae9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:24:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: c18eb5bd-42f0-4ff8-8063-8b1d912cbc04 kind: Event lastTimestamp: "2026-07-13T19:24:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:24Z" name: switch-graph-4dc86.18c1efdaa69ee6d4 namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: 55788b52-fa10-44db-9047-fc84b9e3d47e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-13T19:24:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-4dc86 namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: c18eb5bd-42f0-4ff8-8063-8b1d912cbc04 kind: Event lastTimestamp: "2026-07-13T19:24:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:24:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:24:24Z" name: switch-graph-4dc86.18c1efdaa702fe71 namespace: kserve-ci-e2e-test resourceVersion: "27378" uid: de07b1b3-2d07-4376-b3bc-8f678fdc10d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-13T19:03:08.759970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-91c3d-58989656cf-jcztv to ip-10-0-137-93.ec2.internal metadata: creationTimestamp: "2026-07-13T19:03:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-13T19:03:08Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb50b35986a namespace: kserve-ci-e2e-test resourceVersion: "16557" uid: 25b52944-1142-4b40-b0d1-7a02bdea4c84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bb9bb9746-dtkvf source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:08Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:03:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-91c3d-serving-cert" not found' metadata: creationTimestamp: "2026-07-13T19:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:08Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb514e26715 namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: ee5e5357-34cb-4c91-af65-7783dd60517b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16561" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:03:09Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-13T19:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-13T19:03:09Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb5489a4f2b namespace: kserve-ci-e2e-test resourceVersion: "16577" uid: 7d411122-4b31-4672-92fd-3f82cde87156 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91c3d} kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:03:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:814af323b5d1cbdab913eb2692456d1382c8eaf87a5271e4958bd36fa46947bd" already present on machine metadata: creationTimestamp: "2026-07-13T19:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:09Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb54a3b62f3 namespace: kserve-ci-e2e-test resourceVersion: "16579" uid: 528140b9-2a8a-4f15-a6e7-769c390f340a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91c3d} kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:03:09Z" message: 'Created container: switch-graph-91c3d' metadata: creationTimestamp: "2026-07-13T19:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:09Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb54d5f6f31 namespace: kserve-ci-e2e-test resourceVersion: "16580" uid: 7ded5ea4-6135-495e-bc1d-2a5f3d1cab8b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91c3d} kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:03:09Z" message: Started container switch-graph-91c3d metadata: creationTimestamp: "2026-07-13T19:03:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:03:09Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1eeb54df0f838 namespace: kserve-ci-e2e-test resourceVersion: "16581" uid: 9a8d9cbd-7184-4075-9183-6f5e012904e3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:11:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91c3d} kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:11:23Z" message: Stopping container switch-graph-91c3d metadata: creationTimestamp: "2026-07-13T19:11:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:23Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1ef2837bc3064 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 4913c2bf-cd8e-4a08-bc02-8fb6cdcb00aa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-13T19:11:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-91c3d} kind: Pod name: switch-graph-91c3d-58989656cf-jcztv namespace: kserve-ci-e2e-test resourceVersion: "16554" uid: edceb472-9edf-4aec-950f-ce7da19b2822 kind: Event lastTimestamp: "2026-07-13T19:11:51Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-13T19:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-13T19:11:51Z" name: switch-graph-91c3d-58989656cf-jcztv.18c1ef28f0b5f64a namespace: kserve-ci-e2e-test resourceVersion: "20403" uid: adf386fe-7a8b-4e23-9d70-07bb0d84bf12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-93.ec2.internal source: component: kubelet host: ip-10-0-137-93.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-91c3d-58989656cf namespace: kserve-ci-e2e-test resourceVersion: "16549" uid: ac6808bb-6417-46b5-aed5-b823abb75a27 kind: Event lastTimestamp: "2026-07-13T19:03:08Z" message: 'Created pod: switch-graph-91c3d-58989656cf-jcztv' metadata: creationTimestamp: "2026-07-13T19:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:08Z" name: switch-graph-91c3d-58989656cf.18c1eeb50ab2eb76 namespace: kserve-ci-e2e-test resourceVersion: "16553" uid: d8e41a52-41cb-485a-8ad9-dfa57362fc81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16547" uid: 4df5009a-5af6-4ad2-a4ea-7645a62850f4 kind: Event lastTimestamp: "2026-07-13T19:03:08Z" message: Scaled up replica set switch-graph-91c3d-58989656cf from 0 to 1 metadata: creationTimestamp: "2026-07-13T19:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:08Z" name: switch-graph-91c3d.18c1eeb509e1b7aa namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: a19635db-ae91-41a5-b979-13e8684ed692 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16565" uid: c9344e17-d9be-40f1-83da-61ef0b88f17c kind: Event lastTimestamp: "2026-07-13T19:03:08Z" message: InferenceGraph [switch-graph-91c3d] is Ready metadata: creationTimestamp: "2026-07-13T19:03:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-13T19:03:08Z" name: switch-graph-91c3d.18c1eeb50fdd8b58 namespace: kserve-ci-e2e-test resourceVersion: "16567" uid: ed61c496-f795-4d16-8e15-1ec3a5c10dbb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: cf167696-f452-4ccf-a869-ef67ddac9812 kind: Event lastTimestamp: "2026-07-13T19:03:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:23Z" name: switch-graph-91c3d.18c1eeb888a4bcd3 namespace: kserve-ci-e2e-test resourceVersion: "16703" uid: 6d5a8bcd-c8dc-46f6-8bc5-b2163a1f35d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16548" uid: cf167696-f452-4ccf-a869-ef67ddac9812 kind: Event lastTimestamp: "2026-07-13T19:03:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-13T19:03:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:23Z" name: switch-graph-91c3d.18c1eeb8891ac5ac namespace: kserve-ci-e2e-test resourceVersion: "16705" uid: 51467fab-d023-4486-99db-3a825087edb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: cf167696-f452-4ccf-a869-ef67ddac9812 kind: Event lastTimestamp: "2026-07-13T19:03:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:38Z" name: switch-graph-91c3d.18c1eebc085915c5 namespace: kserve-ci-e2e-test resourceVersion: "16828" uid: 5e59db94-23e0-43a5-954b-aa7d4370bfed reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-13T19:03:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-91c3d namespace: kserve-ci-e2e-test resourceVersion: "16704" uid: cf167696-f452-4ccf-a869-ef67ddac9812 kind: Event lastTimestamp: "2026-07-13T19:03:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-13T19:03:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-13T19:03:38Z" name: switch-graph-91c3d.18c1eebc08c9dd50 namespace: kserve-ci-e2e-test resourceVersion: "16830" uid: 3ee8dbd2-3d9f-4157-9371-3ec986d480ec reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34297"