--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:38:00.790412Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-59e5e-6fc8d76f64-fw7jh to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:38:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:38:00Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c444208107 namespace: kserve-ci-e2e-test resourceVersion: "20463" uid: 61bae54d-c9f4-4d6e-ba56-d570855ad7b9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:01Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:01Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:38:01Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c45db0a37b namespace: kserve-ci-e2e-test resourceVersion: "20480" uid: a324ab9d-5b92-470a-88fa-20d03fd8643a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-59e5e} kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:01Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:01Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c45efe30b7 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: c0832748-7a69-47e8-be9a-72991242cd5a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-59e5e} kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:01Z" message: 'Created container: ensemble-graph-59e5e' metadata: creationTimestamp: "2026-07-28T17:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:01Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c4627e1d45 namespace: kserve-ci-e2e-test resourceVersion: "20485" uid: f58b595c-cf00-4137-808a-e11ba821c4c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-59e5e} kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:01Z" message: Started container ensemble-graph-59e5e metadata: creationTimestamp: "2026-07-28T17:38:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:01Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c462f8bbff namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: 09f1a846-a005-4319-a786-42680107a28f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-59e5e} kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:10Z" message: Stopping container ensemble-graph-59e5e metadata: creationTimestamp: "2026-07-28T17:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:10Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c69ccf0868 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: 9591254b-da45-4ddd-b758-d7cf3a6f225b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-59e5e} kind: Pod name: ensemble-graph-59e5e-6fc8d76f64-fw7jh namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: f1753bc9-60d2-4e29-8a88-175f5d08c3c9 kind: Event lastTimestamp: "2026-07-28T17:38:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:37Z" name: ensemble-graph-59e5e-6fc8d76f64-fw7jh.18c684c6fa67b52d namespace: kserve-ci-e2e-test resourceVersion: "20937" uid: e5cc8e30-acc1-456c-a6b6-6c42d23fff90 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-59e5e-6fc8d76f64 namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: f824cac0-d82f-4bfe-b8ae-9157fd824c8d kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: 'Created pod: ensemble-graph-59e5e-6fc8d76f64-fw7jh' metadata: creationTimestamp: "2026-07-28T17:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: ensemble-graph-59e5e-6fc8d76f64.18c684c443ba841b namespace: kserve-ci-e2e-test resourceVersion: "20459" uid: 578e8070-6753-46e2-8dd2-b54dd90cbf66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-59e5e namespace: kserve-ci-e2e-test resourceVersion: "20448" uid: dac44601-62d3-434b-80a0-6d986025400c kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: Scaled up replica set ensemble-graph-59e5e-6fc8d76f64 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: ensemble-graph-59e5e.18c684c44309aec4 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: ca6aef95-b7b4-4be7-93a0-c9306b5b719a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:00Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-59e5e namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: 5cfe3a50-cba6-4209-a43b-14ae7ed95ac7 kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: InferenceGraph [ensemble-graph-59e5e] is Ready metadata: creationTimestamp: "2026-07-28T17:38:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:00Z" name: ensemble-graph-59e5e.18c684c4485b930c namespace: kserve-ci-e2e-test resourceVersion: "20474" uid: 3f5b060c-74ad-445b-95ef-2d75741aa866 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:39:21.117701Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21510" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-c57a1-76b899c654-jhp92 to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c684d6f801a3aa namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: 50dcdb4e-44e2-4041-aa00-3cb3b66d0257 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c684d7124a2147 namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: cc8df279-2b6c-4344-9fe4-2a030507ebd9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c57a1} kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c684d7138c3cd1 namespace: kserve-ci-e2e-test resourceVersion: "21531" uid: 7e956a98-4231-4185-a9bc-003a9137c63b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c57a1} kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: 'Created container: ensemble-graph-c57a1' metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c684d716775415 namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: e63ca3de-4f0f-4168-b402-378e570cc219 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c57a1} kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: Started container ensemble-graph-c57a1 metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c684d716e62ac6 namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 260c5682-a8e5-4909-a58b-3d01f45975a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c57a1} kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:47:35Z" message: Stopping container ensemble-graph-c57a1 metadata: creationTimestamp: "2026-07-28T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:35Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c6854a269ea9dd namespace: kserve-ci-e2e-test resourceVersion: "25018" uid: 5990ffbd-5e0a-4291-a0e9-4300ca206f66 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-c57a1} kind: Pod name: ensemble-graph-c57a1-76b899c654-jhp92 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: 632b065e-c3a9-4fa7-93aa-b2344af010f4 kind: Event lastTimestamp: "2026-07-28T17:48:02Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:02Z" name: ensemble-graph-c57a1-76b899c654-jhp92.18c6854a9c4291c6 namespace: kserve-ci-e2e-test resourceVersion: "25352" uid: aec6ee86-6651-4dc1-b81f-2fa51118df1e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-c57a1-76b899c654 namespace: kserve-ci-e2e-test resourceVersion: "21507" uid: 38605bb9-6324-4fb4-93c6-c6b5405cf1b8 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: 'Created pod: ensemble-graph-c57a1-76b899c654-jhp92' metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1-76b899c654.18c684d6f794d9aa namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 20bee7e2-3090-4cb1-b593-359edfbbef04 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21505" uid: 9ec9aff6-b0d3-4dce-a1db-a54b36007cde kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: Scaled up replica set ensemble-graph-c57a1-76b899c654 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1.18c684d6f6f6d9c8 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: c7997262-2c0c-4526-880f-306df8dc4c66 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 9de85db8-8c92-4582-9b5d-dab374b5fa27 kind: Event lastTimestamp: "2026-07-28T17:39:21Z" message: InferenceGraph [ensemble-graph-c57a1] is Ready metadata: creationTimestamp: "2026-07-28T17:39:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:21Z" name: ensemble-graph-c57a1.18c684d6fcffd4a6 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: f4b3071c-b712-4f2e-a5b5-baa6e5afb7d8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: c506399e-6dba-4265-acfd-e60b0a7ef28c kind: Event lastTimestamp: "2026-07-28T17:39:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:36Z" name: ensemble-graph-c57a1.18c684da75c6c4e3 namespace: kserve-ci-e2e-test resourceVersion: "21653" uid: 15d6d2fc-07b3-48ea-8dbf-204a7f7caa00 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: c506399e-6dba-4265-acfd-e60b0a7ef28c kind: Event lastTimestamp: "2026-07-28T17:39:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:36Z" name: ensemble-graph-c57a1.18c684da760c69d8 namespace: kserve-ci-e2e-test resourceVersion: "21655" uid: 4ab4df6c-2d12-410e-bc50-f8a6e69f0bcd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: c506399e-6dba-4265-acfd-e60b0a7ef28c kind: Event lastTimestamp: "2026-07-28T17:39:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:51Z" name: ensemble-graph-c57a1.18c684ddf4dccf58 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: fd5fbe6f-13b6-4c4d-af8f-932e8d3216c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: c506399e-6dba-4265-acfd-e60b0a7ef28c kind: Event lastTimestamp: "2026-07-28T17:39:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:51Z" name: ensemble-graph-c57a1.18c684ddf546e664 namespace: kserve-ci-e2e-test resourceVersion: "21780" uid: 16602159-762a-4dea-898a-70557fdcd955 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:47:36.182313Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25098" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a3c282c18 namespace: kserve-ci-e2e-test resourceVersion: "25102" uid: 5e5c9cd8-3738-4c4f-bb3f-a7ee13118287 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-32bdd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a4548a9da namespace: kserve-ci-e2e-test resourceVersion: "25113" uid: b407a223-36b1-4fa9-b97a-b53ce35fa94e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25106" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a796af7a7 namespace: kserve-ci-e2e-test resourceVersion: "25131" uid: b07f3b21-3b3c-4f56-b9d1-2b62b3734a8c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a7aecbc4b namespace: kserve-ci-e2e-test resourceVersion: "25133" uid: 9edd476d-9517-4bbb-bd66-186f2c83ba91 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a7dbf6c73 namespace: kserve-ci-e2e-test resourceVersion: "25134" uid: a19e7c77-1366-42a1-b25e-9a6c2a1b69a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a7e32ae37 namespace: kserve-ci-e2e-test resourceVersion: "25136" uid: 67f478b5-6996-4193-8fce-caa3afa682dc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a7e3cf7aa namespace: kserve-ci-e2e-test resourceVersion: "25137" uid: 5b8ef303-5882-471f-9c69-cae09cf93dcb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a80e6b290 namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: d23b22e1-8af0-4c51-88f8-60f2d553111e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:47:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:37Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a81676a13 namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: f4cd07a9-b2b1-4e5d-9410-348257f07e12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:47:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:48:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:47:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:13Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6854a88847d25 namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: 939247da-2776-4bd2-ad0b-abbe0b14e658 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6855a8be2b181 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 49e77da5-426b-4b0c-a295-152a95394ff0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6855a8be33844 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 63a35acb-c585-49a4-b0e5-4a81e00392e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8 namespace: kserve-ci-e2e-test resourceVersion: "25100" uid: 1c23fd96-bc7e-4393-8a9d-e5466324a746 kind: Event lastTimestamp: "2026-07-28T17:48:48Z" message: 'Readiness probe failed: Get "https://10.132.0.37:8643/healthz": dial tcp 10.132.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:48Z" name: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8.18c6855b10b47fdd namespace: kserve-ci-e2e-test resourceVersion: "26330" uid: 8fa19ecb-7a85-41d6-b286-ea55915441e9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-32bdd-predictor-84cb75477b namespace: kserve-ci-e2e-test resourceVersion: "25092" uid: 42f9e366-611b-4767-888c-d86be7a0f250 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: 'Created pod: error-404-isvc-32bdd-predictor-84cb75477b-gsdp8' metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-32bdd-predictor-84cb75477b.18c6854a3bacb18d namespace: kserve-ci-e2e-test resourceVersion: "25099" uid: 1b433c22-6010-499b-8cea-372c1b9ee261 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: d78b649f-9b4d-487f-8506-04e366a7ed25 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Scaled up replica set error-404-isvc-32bdd-predictor-84cb75477b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-32bdd-predictor.18c6854a396f85f3 namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: ca20bafa-9700-4013-855e-db3dbf690506 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:47:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 2a84fe00-6bc4-4929-8abd-7ba125836e48 kind: Event lastTimestamp: "2026-07-28T17:48:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:06Z" name: error-404-isvc-32bdd-predictor.18c6854db7ecc4c3 namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 673e4567-27e8-4a6b-9c2e-c73ddbe65043 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:47:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25091" uid: 2a84fe00-6bc4-4929-8abd-7ba125836e48 kind: Event lastTimestamp: "2026-07-28T17:48:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:06Z" name: error-404-isvc-32bdd-predictor.18c6854db853d832 namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 8c98948c-48a6-45d2-b0da-ce28e3d786ae reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 2a84fe00-6bc4-4929-8abd-7ba125836e48 kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: error-404-isvc-32bdd-predictor.18c68554b5f768db namespace: kserve-ci-e2e-test resourceVersion: "26087" uid: 2b19a7a4-4913-4697-8b3c-291407da375c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 2a84fe00-6bc4-4929-8abd-7ba125836e48 kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: error-404-isvc-32bdd-predictor.18c68554b6728ee6 namespace: kserve-ci-e2e-test resourceVersion: "26088" uid: 847ad34a-1db8-416c-97d0-e5e497491805 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32bdd namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: 9152efa9-7490-480e-8609-bd51be5c9cd2 kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-32bdd.18c6854a397bcecb namespace: kserve-ci-e2e-test resourceVersion: "25846" uid: 35094a03-bb30-4990-8897-04a6045f5e03 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-32bdd namespace: kserve-ci-e2e-test resourceVersion: "25843" uid: 9152efa9-7490-480e-8609-bd51be5c9cd2 kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: InferenceService [error-404-isvc-32bdd] is Ready metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-32bdd.18c6855543f376fb namespace: kserve-ci-e2e-test resourceVersion: "25844" uid: 7e4bc9cb-927e-4bab-a2a7-0084da265c4c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:37:56.870362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c35a795dbf namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: 5ce5ece9-a5a9-484d-b55f-b86f1ce67c5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-4108d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c3664c53b8 namespace: kserve-ci-e2e-test resourceVersion: "20379" uid: 7ce244ea-2e61-4134-9e29-c7a0c322328f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c397c37a0b namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: 230ff359-86ea-4e9f-bd7d-c65de250125f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c39929f64f namespace: kserve-ci-e2e-test resourceVersion: "20401" uid: b95e4da7-cd81-424e-88c2-6fdff53c27e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c39c84f1d5 namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 268b68af-3f69-440d-9957-bae381594dda reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c39cf856a9 namespace: kserve-ci-e2e-test resourceVersion: "20404" uid: 339af657-1082-4225-96fa-2909a190b3ab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c39d034b80 namespace: kserve-ci-e2e-test resourceVersion: "20405" uid: 4799557a-913a-4047-8e11-a1b05f47f0fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:58Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c3a0413d98 namespace: kserve-ci-e2e-test resourceVersion: "20406" uid: 9ec36bf6-5f88-423b-9afc-0737048e6b46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:37:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:58Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c3a0b8d1ac namespace: kserve-ci-e2e-test resourceVersion: "20407" uid: a93809fc-2f51-43b4-a9df-e1ebcae78090 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:37:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:38:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:44Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684c3a89eb548 namespace: kserve-ci-e2e-test resourceVersion: "20992" uid: 51cffc8a-d6f7-4b90-94b9-4da0ce36eb02 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684d3b19cf388 namespace: kserve-ci-e2e-test resourceVersion: "21295" uid: b1b89585-f52e-4124-938c-1a264acd7e88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684d3b19e36f2 namespace: kserve-ci-e2e-test resourceVersion: "21298" uid: dd47a390-af57-4129-a870-a4232c1279b0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx namespace: kserve-ci-e2e-test resourceVersion: "20363" uid: 2a7ef913-e7c5-4a2c-9b8a-1b2ec72cfe16 kind: Event lastTimestamp: "2026-07-28T17:39:09Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:09Z" name: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx.18c684d430bfa2e0 namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: 01abc1df-1583-4e50-be40-699c72323b71 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4108d-predictor-7bb6fdb996 namespace: kserve-ci-e2e-test resourceVersion: "20354" uid: ee57959e-0d28-43ec-9e05-7fcfcf7f05e1 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'Created pod: error-404-isvc-4108d-predictor-7bb6fdb996-q7wdx' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-4108d-predictor-7bb6fdb996.18c684c359e820de namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 7098d11f-c890-455c-834d-bd7ad90d2bcb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20352" uid: dcf292bf-8a0e-4659-8e7a-122cfb482435 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Scaled up replica set error-404-isvc-4108d-predictor-7bb6fdb996 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-4108d-predictor.18c684c35824edf6 namespace: kserve-ci-e2e-test resourceVersion: "20355" uid: 06ffc680-52d0-4944-9c1d-261c4c4787c1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 803c2ef2-ecd1-48f4-afba-2b0ac3e6560a kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-4108d-predictor.18c684c6d6be9414 namespace: kserve-ci-e2e-test resourceVersion: "20707" uid: ddfaf8a3-1d47-4836-bafa-f946b477dac5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20353" uid: 803c2ef2-ecd1-48f4-afba-2b0ac3e6560a kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-4108d-predictor.18c684c6d7400d9a namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: b3d034dd-ab30-4cc8-b64c-e328cbbb8e62 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 803c2ef2-ecd1-48f4-afba-2b0ac3e6560a kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: error-404-isvc-4108d-predictor.18c684ca55e5e307 namespace: kserve-ci-e2e-test resourceVersion: "21136" uid: 3b57b030-3de4-4ace-a91b-0811c3a7cb7b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: 803c2ef2-ecd1-48f4-afba-2b0ac3e6560a kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: error-404-isvc-4108d-predictor.18c684ca564aefc3 namespace: kserve-ci-e2e-test resourceVersion: "21138" uid: f28e721c-e163-4dd5-95f9-de1dd7b6bb1a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 93d61ed7-8c50-415f-9f58-4df3300a7f81 kind: Event lastTimestamp: "2026-07-28T17:38:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:54Z" name: error-404-isvc-4108d.18c684c3582a27ec namespace: kserve-ci-e2e-test resourceVersion: "21083" uid: 1d046965-b336-4d35-ab49-d181232f9bc4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 93d61ed7-8c50-415f-9f58-4df3300a7f81 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-4108d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4108d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-4108d.18c684c3588734f1 namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: 5b3ef85a-ed9b-4561-a18b-51b655699ec5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20343" uid: 93d61ed7-8c50-415f-9f58-4df3300a7f81 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4108d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-4108d.18c684c35887960f namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 3f8ccb73-d357-4035-b220-2682e996001e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "21081" uid: 93d61ed7-8c50-415f-9f58-4df3300a7f81 kind: Event lastTimestamp: "2026-07-28T17:38:54Z" message: InferenceService [error-404-isvc-4108d] is Ready metadata: creationTimestamp: "2026-07-28T17:38:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:54Z" name: error-404-isvc-4108d.18c684d0b7fe5d2e namespace: kserve-ci-e2e-test resourceVersion: "21082" uid: febd3d0e-19ca-4514-9490-409a42c18137 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:37:00.847692Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19774" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-59e5e-predictor-5468b8979b-jxhfx to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b64f435707 namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: 3b132da1-115c-47b2-aaad-329da242f875 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-59e5e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b65e1c1095 namespace: kserve-ci-e2e-test resourceVersion: "19793" uid: 6bf244b9-f623-4a4a-8963-adfa857ae42c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19785" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b68c9b49e4 namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: 607a9324-bce3-4752-8491-0f46d1cb09b9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b68df946ae namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 51bb8551-78af-46df-bb76-1ddd579a5b5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b6920f6c68 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: 2a22a865-5c41-49f3-8188-cc094068f969 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b6928ab07d namespace: kserve-ci-e2e-test resourceVersion: "19820" uid: 26e10125-2370-4b8e-afab-f4e30c95fd6b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b69295ebda namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: d413e958-5df5-48ee-8824-b5c818bacf7b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:02Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:02Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b696659b56 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: 2d4ad57a-6216-442a-95b8-d32ed669aae3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:02Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:02Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b696d936de namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 797b5b04-c7de-4fdb-bf46-19765a25c5ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:37:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:38Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684b6cc292c52 namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 3ed4853b-ff42-400d-9315-fe055b8b691d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:38:10Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:10Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684c6a31927dc namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: df87ecf5-d3be-48a6-a9e8-5840d925d6ae reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:38:10Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:38:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:10Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684c6a31f8c79 namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 783f2441-3607-404b-85d4-f23638f6aa42 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 88e90ec2-a3f3-45b5-86eb-233b4a64cff3 kind: Event lastTimestamp: "2026-07-28T17:38:13Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:38:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:13Z" name: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx.18c684c754a27c5a namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: aca62c62-bc0a-4547-859a-20152dfac730 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-59e5e-predictor-5468b8979b namespace: kserve-ci-e2e-test resourceVersion: "19766" uid: 56b9fd0a-cbba-4d68-8ec0-e44e4ed12e45 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: 'Created pod: error-404-isvc-59e5e-predictor-5468b8979b-jxhfx' metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-59e5e-predictor-5468b8979b.18c684b64ed4db2a namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 02435eee-6a0e-4232-bc37-9dc93b8876ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19764" uid: bb6d7bf8-4467-44ab-b35a-bc8bd8e5f2d2 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Scaled up replica set error-404-isvc-59e5e-predictor-5468b8979b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-59e5e-predictor.18c684b64d427e8e namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: ddb3e674-a176-4a85-bfd5-d27b6905c62a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 4bdb7e01-8e58-4bc9-b343-76ac7d34abbf kind: Event lastTimestamp: "2026-07-28T17:37:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:30Z" name: error-404-isvc-59e5e-predictor.18c684b9cbbfcee2 namespace: kserve-ci-e2e-test resourceVersion: "20063" uid: 40f7c705-2c5f-4f3d-b8b9-a7ed4d95092c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19765" uid: 4bdb7e01-8e58-4bc9-b343-76ac7d34abbf kind: Event lastTimestamp: "2026-07-28T17:37:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:30Z" name: error-404-isvc-59e5e-predictor.18c684b9cc290047 namespace: kserve-ci-e2e-test resourceVersion: "20064" uid: b989076f-5b20-4d85-8f20-dab2f191f3a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 4bdb7e01-8e58-4bc9-b343-76ac7d34abbf kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: error-404-isvc-59e5e-predictor.18c684c0c9780e8f namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: 695c6d46-0c6e-4c93-8d08-1bc5f1f782ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: 4bdb7e01-8e58-4bc9-b343-76ac7d34abbf kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: error-404-isvc-59e5e-predictor.18c684c0c9de501a namespace: kserve-ci-e2e-test resourceVersion: "20476" uid: 555bd3f8-0362-43c0-8170-6edc46cd0592 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 65ee5d2f-5463-4339-a085-e169f6a2008b kind: Event lastTimestamp: "2026-07-28T17:37:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:49Z" name: error-404-isvc-59e5e.18c684b64d47eb18 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: ff869c16-478c-4d2c-b7c7-0327c66e36f2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 65ee5d2f-5463-4339-a085-e169f6a2008b kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: 'Failed to update status for InferenceService "error-404-isvc-59e5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-59e5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-59e5e.18c684b64de798d4 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 88a863f6-0997-40bf-a48c-7ecf4e26c2b9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "19745" uid: 65ee5d2f-5463-4339-a085-e169f6a2008b kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-59e5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-59e5e.18c684b64de80b51 namespace: kserve-ci-e2e-test resourceVersion: "19773" uid: e2066bab-7077-4958-85d2-d2524a9ce718 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "20213" uid: 65ee5d2f-5463-4339-a085-e169f6a2008b kind: Event lastTimestamp: "2026-07-28T17:37:49Z" message: InferenceService [error-404-isvc-59e5e] is Ready metadata: creationTimestamp: "2026-07-28T17:37:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:49Z" name: error-404-isvc-59e5e.18c684c18745e874 namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: e84a181b-2d82-440f-97f5-4add50309046 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:27:46.003016Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-65026-predictor-7b8556fc9b-dkthh to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:27:46Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684351ff15d11 namespace: kserve-ci-e2e-test resourceVersion: "15388" uid: abe8ed10-da70-4a87-a92b-e2e2e6d8ad85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-65026-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:46Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684352c90f6cd namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 52abbf8f-78b1-4bd1-9c88-033317af5ef3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15394" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684355e3222ae namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: f1eed7b6-d34d-4dbe-ae3d-357dfea683f6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684355fc69fa6 namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: c35265ea-b009-4876-9cf4-e20c85969de6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c68435637167ba namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: 8e282db9-c1cb-4a32-b109-23aab1bacdb1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c6843563e2185c namespace: kserve-ci-e2e-test resourceVersion: "15426" uid: 2d2e41be-9cd4-4a34-9d1b-8770f69197f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c6843563eda19c namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 1c09a439-5aa9-4498-97f4-349246dabf92 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684356688b707 namespace: kserve-ci-e2e-test resourceVersion: "15429" uid: 358bf640-6990-47f7-8b0b-ec257ed8142b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c6843566ef45ec namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: 728e1127-fcb0-4b41-9b9e-e6659c9b9f3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:37:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:03Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684358bd78f40 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: 04b5a190-aeec-4dc4-a4f3-1550ffc192eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684b648f9c002 namespace: kserve-ci-e2e-test resourceVersion: "19744" uid: be7c1f57-4124-450b-b6dd-0542def22ef8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:00Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684b648fa3f96 namespace: kserve-ci-e2e-test resourceVersion: "19746" uid: 49a272f9-e02c-4750-929e-63f56a57418e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7317bca5-141f-480b-8e65-1fba4a9b23ab kind: Event lastTimestamp: "2026-07-28T17:37:03Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:03Z" name: error-404-isvc-65026-predictor-7b8556fc9b-dkthh.18c684b700378265 namespace: kserve-ci-e2e-test resourceVersion: "19843" uid: 13318dbb-d631-46a8-84e4-339bd9764b36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-65026-predictor-7b8556fc9b namespace: kserve-ci-e2e-test resourceVersion: "15374" uid: 265aeaf9-b55e-4111-a251-a9f4e1abc3a9 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: 'Created pod: error-404-isvc-65026-predictor-7b8556fc9b-dkthh' metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-65026-predictor-7b8556fc9b.18c684351f7ed8a5 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 5e1dc684-7ebe-473a-8516-39fe53db247f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15372" uid: d8b87dc2-d2c4-498c-9249-69433e80cfed kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Scaled up replica set error-404-isvc-65026-predictor-7b8556fc9b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-65026-predictor.18c684351dbbdccc namespace: kserve-ci-e2e-test resourceVersion: "15376" uid: 3704bb00-f27d-4dcf-b224-ec88155a551e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 02ad0acd-75aa-42a6-af93-650a212e83e6 kind: Event lastTimestamp: "2026-07-28T17:28:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:15Z" name: error-404-isvc-65026-predictor.18c684389c484b76 namespace: kserve-ci-e2e-test resourceVersion: "15708" uid: 1df5612d-31df-476f-86c4-9a7f2aebff08 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 02ad0acd-75aa-42a6-af93-650a212e83e6 kind: Event lastTimestamp: "2026-07-28T17:28:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:16Z" name: error-404-isvc-65026-predictor.18c684389cb5d163 namespace: kserve-ci-e2e-test resourceVersion: "15709" uid: 2e09f851-02c3-42ad-b4b3-0d4455da3bd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 02ad0acd-75aa-42a6-af93-650a212e83e6 kind: Event lastTimestamp: "2026-07-28T17:28:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:46Z" name: error-404-isvc-65026-predictor.18c6843f9a408cf5 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: b20d4678-8774-4719-b924-4bc6ca8243fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15568" uid: 02ad0acd-75aa-42a6-af93-650a212e83e6 kind: Event lastTimestamp: "2026-07-28T17:28:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:46Z" name: error-404-isvc-65026-predictor.18c6843f9ad13559 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: ee9577e7-0b17-40d6-8fe0-3b3e80ace919 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 5d85b768-44de-4b2e-9ea1-c009fba316a9 kind: Event lastTimestamp: "2026-07-28T17:28:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:34Z" name: error-404-isvc-65026.18c684351dbd512b namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: 71ae64cb-be0d-4a7e-86e6-90319c2c0bb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 5d85b768-44de-4b2e-9ea1-c009fba316a9 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-65026": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65026": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-65026.18c684351e32e1bc namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 1a1c5253-e15e-4e36-bf42-07faaec30c8a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15352" uid: 5d85b768-44de-4b2e-9ea1-c009fba316a9 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-65026": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-65026.18c684351e333f46 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: ea132919-be2e-48b9-a067-56abc76377c4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15897" uid: 5d85b768-44de-4b2e-9ea1-c009fba316a9 kind: Event lastTimestamp: "2026-07-28T17:28:33Z" message: InferenceService [error-404-isvc-65026] is Ready metadata: creationTimestamp: "2026-07-28T17:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:33Z" name: error-404-isvc-65026.18c6844048104ac5 namespace: kserve-ci-e2e-test resourceVersion: "15898" uid: 1068e423-0f3d-4f9f-ad1b-1adb769ef0d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:48:46.472649Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26272" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-80e89-predictor-f86d55dd6-56mdh to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855a99c99659 namespace: kserve-ci-e2e-test resourceVersion: "26276" uid: 19f73d6e-94f4-4243-bb06-61b47e81a5a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-80e89-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855aa3f23826 namespace: kserve-ci-e2e-test resourceVersion: "26289" uid: cb54983d-8b98-4546-857d-a1a47fbb3fa3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855ad7565064 namespace: kserve-ci-e2e-test resourceVersion: "26307" uid: 3dbae19f-36e6-4301-ab5d-9b28d5fb374a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855ad8fa27d3 namespace: kserve-ci-e2e-test resourceVersion: "26310" uid: 1a136501-5f64-4b69-85a2-be5f1e63e22d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855adc04f1a0 namespace: kserve-ci-e2e-test resourceVersion: "26312" uid: adc21e84-ac5f-4e05-9eed-4371d8ccb62c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855adc765627 namespace: kserve-ci-e2e-test resourceVersion: "26313" uid: c953e5fb-cc55-46e6-8d4f-da01f9ecf24d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855adc815d48 namespace: kserve-ci-e2e-test resourceVersion: "26314" uid: 28697a55-605e-4341-ac90-afb4d4dd8b0f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855adf8a20cb namespace: kserve-ci-e2e-test resourceVersion: "26315" uid: 615d3f48-8e1c-458f-94ad-2aea5d382d67 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:48:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:47Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855ae0028645 namespace: kserve-ci-e2e-test resourceVersion: "26316" uid: 298e7cb4-a3d0-4244-b10d-c8ad2e362f8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:58:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:13Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c6855ae6f3af80 namespace: kserve-ci-e2e-test resourceVersion: "30483" uid: afa6f57a-7ae2-455e-ac06-48b7c48ffeb5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:58:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:11Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c685de1b291b57 namespace: kserve-ci-e2e-test resourceVersion: "30462" uid: ed384a23-a772-4496-850d-f1fe757fe1fa reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:58:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:11Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c685de1b2984c1 namespace: kserve-ci-e2e-test resourceVersion: "30463" uid: fea30fae-3402-4138-91c5-f8da4eac96fe reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 23af64eb-22ef-421f-8f81-536bada20b4e kind: Event lastTimestamp: "2026-07-28T17:58:13Z" message: 'Readiness probe failed: Get "https://10.132.0.41:8643/healthz": dial tcp 10.132.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:13Z" name: error-404-isvc-80e89-predictor-f86d55dd6-56mdh.18c685deaf6b1b88 namespace: kserve-ci-e2e-test resourceVersion: "30481" uid: 2aab765d-8517-4800-8e95-8837090f29d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-80e89-predictor-f86d55dd6 namespace: kserve-ci-e2e-test resourceVersion: "26265" uid: c5e10c26-f122-4cc9-92f5-bb36b8938ec2 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Created pod: error-404-isvc-80e89-predictor-f86d55dd6-56mdh' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89-predictor-f86d55dd6.18c6855a99446394 namespace: kserve-ci-e2e-test resourceVersion: "26274" uid: cc3d8399-0d45-4ba4-987d-d47385fe2c60 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26262" uid: da9cc2c4-b732-4404-95e1-2c7ee7411d5f kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Scaled up replica set error-404-isvc-80e89-predictor-f86d55dd6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89-predictor.18c6855a9760bfa2 namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 9f116dd4-d603-4e0a-b6eb-9d5563f9c508 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: ddaa21f1-ad61-4d69-bfc8-06c2f3c0120c kind: Event lastTimestamp: "2026-07-28T17:49:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:01Z" name: error-404-isvc-80e89-predictor.18c6855e15bcce23 namespace: kserve-ci-e2e-test resourceVersion: "26457" uid: 3e1070ea-df15-4685-b67e-8c91b5d629d9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26264" uid: ddaa21f1-ad61-4d69-bfc8-06c2f3c0120c kind: Event lastTimestamp: "2026-07-28T17:49:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:01Z" name: error-404-isvc-80e89-predictor.18c6855e16508277 namespace: kserve-ci-e2e-test resourceVersion: "26459" uid: f6aabacb-2589-4468-a06c-b7b00d000e4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: ddaa21f1-ad61-4d69-bfc8-06c2f3c0120c kind: Event lastTimestamp: "2026-07-28T17:50:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:01Z" name: error-404-isvc-80e89-predictor.18c6856194f6ad7c namespace: kserve-ci-e2e-test resourceVersion: "27023" uid: 886925dd-a731-4ca8-bd04-e23f06b8b0eb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:49:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26458" uid: ddaa21f1-ad61-4d69-bfc8-06c2f3c0120c kind: Event lastTimestamp: "2026-07-28T17:50:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:01Z" name: error-404-isvc-80e89-predictor.18c685619567def7 namespace: kserve-ci-e2e-test resourceVersion: "27025" uid: 274b36f6-01bc-41b3-a121-c4032b69a5e2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 76f86f4c-5681-4330-8d16-5d543ef3f4ce kind: Event lastTimestamp: "2026-07-28T17:49:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:44Z" name: error-404-isvc-80e89.18c6855a976c5b9e namespace: kserve-ci-e2e-test resourceVersion: "26859" uid: 74460c92-29d1-47fc-b103-37bb1db26818 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 76f86f4c-5681-4330-8d16-5d543ef3f4ce kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Failed to update status for InferenceService "error-404-isvc-80e89": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-80e89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89.18c6855a97e9234f namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: 7435dc35-08cb-4fd3-98db-6b95a5143406 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26252" uid: 76f86f4c-5681-4330-8d16-5d543ef3f4ce kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-80e89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:46Z" name: error-404-isvc-80e89.18c6855a97e97977 namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: d07e9495-3019-4224-bd2a-7b9b0775a631 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26853" uid: 76f86f4c-5681-4330-8d16-5d543ef3f4ce kind: Event lastTimestamp: "2026-07-28T17:49:43Z" message: InferenceService [error-404-isvc-80e89] is Ready metadata: creationTimestamp: "2026-07-28T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:43Z" name: error-404-isvc-80e89.18c68567f7e6ce1d namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 30e5138e-6a34-402a-ad5c-96554ef3d438 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:28:41.978722Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-8c20d-predictor-5595b88c76-cj8kf to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:28:41Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c68442285ad85d namespace: kserve-ci-e2e-test resourceVersion: "16080" uid: a34f931f-7b3d-454f-af40-0d145b73c90c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-8c20d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c6844233caf40c namespace: kserve-ci-e2e-test resourceVersion: "16094" uid: 36ee760c-74a7-4b94-a8d3-28d54666dbf0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c6844265b5f442 namespace: kserve-ci-e2e-test resourceVersion: "16110" uid: e6866d4d-3a2b-4dc6-8061-d584f9ad18ec reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c6844267481218 namespace: kserve-ci-e2e-test resourceVersion: "16113" uid: a42f29d6-aaac-4a60-abb0-ce5a4d1ea540 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c6844269f7abef namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: fdb1f0d1-05a1-4ed6-9416-d0388b4e224b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684426a5d0cef namespace: kserve-ci-e2e-test resourceVersion: "16118" uid: 4a0f03a8-6bc1-4c47-acf8-778299664e0f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684426a66b2cb namespace: kserve-ci-e2e-test resourceVersion: "16119" uid: a44b85bd-c317-4ec9-b599-d8e6b1a44401 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684426d4e3485 namespace: kserve-ci-e2e-test resourceVersion: "16120" uid: 2214b4b9-20b0-4208-b211-48963fdbd3f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684426db40545 namespace: kserve-ci-e2e-test resourceVersion: "16121" uid: 2831339d-6771-46e6-a695-6e8c9f3bad99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:28:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:29:21Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:21Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c68442df18b451 namespace: kserve-ci-e2e-test resourceVersion: "16483" uid: edf2b308-4938-436f-b3de-fb4b5dd48d45 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684c34deec121 namespace: kserve-ci-e2e-test resourceVersion: "20298" uid: 89998907-59ed-4dce-a8db-53e7f74e81be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: a1eb7223-5e8a-4e06-a5d2-36fd81c306d5 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:56Z" name: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf.18c684c34def50bd namespace: kserve-ci-e2e-test resourceVersion: "20299" uid: bb945a62-b4ec-4196-a45b-76d4bbeec5fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-8c20d-predictor-5595b88c76 namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: dfc4ed4b-949e-4b97-a8b9-419deaa4199a kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'Created pod: error-404-isvc-8c20d-predictor-5595b88c76-cj8kf' metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: error-404-isvc-8c20d-predictor-5595b88c76.18c6844227bb7276 namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: 51c0187e-8a25-4a6c-bc59-2948fdc79e50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: 5312f2ab-a414-4b3b-a9cf-4b25b0197330 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Scaled up replica set error-404-isvc-8c20d-predictor-5595b88c76 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: error-404-isvc-8c20d-predictor.18c6844226124447 namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: 85298430-6e04-404f-a4d6-b0f56794f047 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 64b0ef5d-e42a-47d7-8cb9-3f5d624687e2 kind: Event lastTimestamp: "2026-07-28T17:28:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:56Z" name: error-404-isvc-8c20d-predictor.18c68445a493354e namespace: kserve-ci-e2e-test resourceVersion: "16297" uid: 758f6c45-df03-410c-a1d3-2d4e5ebce597 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16065" uid: 64b0ef5d-e42a-47d7-8cb9-3f5d624687e2 kind: Event lastTimestamp: "2026-07-28T17:28:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:56Z" name: error-404-isvc-8c20d-predictor.18c68445a4f7487b namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: f6d58378-44b7-45c5-ac74-a375d941b2e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 64b0ef5d-e42a-47d7-8cb9-3f5d624687e2 kind: Event lastTimestamp: "2026-07-28T17:30:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:30:12Z" name: error-404-isvc-8c20d-predictor.18c6844923b46782 namespace: kserve-ci-e2e-test resourceVersion: "16909" uid: 171c3071-8e83-4534-92bd-4aff41a73414 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:29:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 64b0ef5d-e42a-47d7-8cb9-3f5d624687e2 kind: Event lastTimestamp: "2026-07-28T17:30:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:30:12Z" name: error-404-isvc-8c20d-predictor.18c68449241c5166 namespace: kserve-ci-e2e-test resourceVersion: "16910" uid: d025d232-0d71-4571-8253-dc80f50d8cd4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 4ad26a04-42b1-4ea2-b1c9-da2ca5983f7e kind: Event lastTimestamp: "2026-07-28T17:29:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:29:31Z" name: error-404-isvc-8c20d.18c684422616dda1 namespace: kserve-ci-e2e-test resourceVersion: "16574" uid: db21524a-55b6-4ef3-8f5b-0345f8872f7e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 4ad26a04-42b1-4ea2-b1c9-da2ca5983f7e kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-8c20d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8c20d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:41Z" name: error-404-isvc-8c20d.18c6844226e06129 namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 0e2a1467-1c36-4e06-ad9c-dbcaf6d625c8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16032" uid: 4ad26a04-42b1-4ea2-b1c9-da2ca5983f7e kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-8c20d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:41Z" name: error-404-isvc-8c20d.18c6844226e0ec1a namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: b9a9ea1a-a9e8-4464-bd3e-a133e845e03a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16572" uid: 4ad26a04-42b1-4ea2-b1c9-da2ca5983f7e kind: Event lastTimestamp: "2026-07-28T17:29:31Z" message: InferenceService [error-404-isvc-8c20d] is Ready metadata: creationTimestamp: "2026-07-28T17:29:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:29:31Z" name: error-404-isvc-8c20d.18c6844d9a6a0daf namespace: kserve-ci-e2e-test resourceVersion: "16573" uid: 2f04df76-3f18-483a-baa4-c1b37216095b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:48:21.960209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25735" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c68554e4bb73d6 namespace: kserve-ci-e2e-test resourceVersion: "25739" uid: 0db94f71-3c35-4e2e-a975-6c8be98bfaf0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c0ed1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c68554f4e2242c namespace: kserve-ci-e2e-test resourceVersion: "25779" uid: 2fd85943-4113-439a-ad0f-bd18a9d5e97d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25743" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c685552378ce97 namespace: kserve-ci-e2e-test resourceVersion: "25801" uid: 8ef9761c-d987-423f-9cee-722bc0eacc17 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c6855524ec3248 namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: d05d0621-0739-4679-83b1-f41f3128619a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c685552802091b namespace: kserve-ci-e2e-test resourceVersion: "25829" uid: 17426aba-1070-489e-8e68-e01ba37cf975 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c685552879f889 namespace: kserve-ci-e2e-test resourceVersion: "25830" uid: 97302a83-bd49-49cd-ad83-1d8247186b12 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c6855528848d1f namespace: kserve-ci-e2e-test resourceVersion: "25831" uid: 63512a4e-76af-4423-b9f6-2130b02b338b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c685552bd82158 namespace: kserve-ci-e2e-test resourceVersion: "25832" uid: ac3fd3de-8275-4dfb-a4ff-269ca3f26787 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:48:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:23Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c685552c4b7f4f namespace: kserve-ci-e2e-test resourceVersion: "25833" uid: c5806a40-9205-4f99-a5da-7b8d082d096f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T17:49:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:09Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c6855549b51dac namespace: kserve-ci-e2e-test resourceVersion: "26519" uid: c6c9f006-4ef0-415f-bfa0-5db0eb4244fe reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T18:05:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:51Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c6864941999a84 namespace: kserve-ci-e2e-test resourceVersion: "33656" uid: e5337957-61b3-47e4-b981-720cf5ff8f20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T18:05:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:51Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c686494199fdc1 namespace: kserve-ci-e2e-test resourceVersion: "33657" uid: 245ebb94-9f62-4d55-b65b-0dde40b072ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps namespace: kserve-ci-e2e-test resourceVersion: "25737" uid: 7df2b1d2-0e0c-47a2-b152-e21b00bc8e1a kind: Event lastTimestamp: "2026-07-28T18:05:55Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-07-28T18:05:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:55Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps.18c6864a3a0d3d0d namespace: kserve-ci-e2e-test resourceVersion: "33726" uid: a5cf4e99-c58f-4460-bcba-9d5a29699b0f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c0ed1-predictor-cdc75bd96 namespace: kserve-ci-e2e-test resourceVersion: "25728" uid: 7937ccc6-002a-458c-a563-058cc712af27 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'Created pod: error-404-isvc-c0ed1-predictor-cdc75bd96-gt2ps' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c0ed1-predictor-cdc75bd96.18c68554e42b0230 namespace: kserve-ci-e2e-test resourceVersion: "25736" uid: 18ae5529-b343-4c91-bfbc-0cc0dda95db9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25726" uid: 5a812beb-7eda-47bc-9c01-ae2f2f791381 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Scaled up replica set error-404-isvc-c0ed1-predictor-cdc75bd96 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c0ed1-predictor.18c68554e2572774 namespace: kserve-ci-e2e-test resourceVersion: "25729" uid: d3ade8f9-ba55-452f-9720-e23f99f1cb4f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 2525f579-5d41-4c9a-be42-6d397322f11b kind: Event lastTimestamp: "2026-07-28T17:48:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:51Z" name: error-404-isvc-c0ed1-predictor.18c6855860e3a2a3 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: ad327413-1a91-47c4-9ae5-21bbc7c631ba reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25727" uid: 2525f579-5d41-4c9a-be42-6d397322f11b kind: Event lastTimestamp: "2026-07-28T17:48:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:51Z" name: error-404-isvc-c0ed1-predictor.18c68558614dfcea namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: 92ba37b6-d75f-42c0-aa6a-473d1d48c5d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:49:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: 2525f579-5d41-4c9a-be42-6d397322f11b kind: Event lastTimestamp: "2026-07-28T17:49:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:52Z" name: error-404-isvc-c0ed1-predictor.18c6855f5efaa55c namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 81ddc8ef-83b4-4e17-99c2-7692dc4cbd1c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:49:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26101" uid: 2525f579-5d41-4c9a-be42-6d397322f11b kind: Event lastTimestamp: "2026-07-28T17:49:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:52Z" name: error-404-isvc-c0ed1-predictor.18c6855f5f606970 namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: 082452c0-b607-4887-91ae-02742c09554d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 641d2e0c-2399-4bdb-a3d9-24baf08bf0a6 kind: Event lastTimestamp: "2026-07-28T17:49:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:19Z" name: error-404-isvc-c0ed1.18c68554e269e4b5 namespace: kserve-ci-e2e-test resourceVersion: "26621" uid: f0b507a7-9f43-4bf7-90c1-2e749b1dde61 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 641d2e0c-2399-4bdb-a3d9-24baf08bf0a6 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-c0ed1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c0ed1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c0ed1.18c68554e3019e6e namespace: kserve-ci-e2e-test resourceVersion: "25732" uid: 7f6e5ad8-dd64-473e-a998-7961bc5cfbd3 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25713" uid: 641d2e0c-2399-4bdb-a3d9-24baf08bf0a6 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c0ed1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c0ed1.18c68554e30206ca namespace: kserve-ci-e2e-test resourceVersion: "25734" uid: 3d17a496-ca3c-4476-b8ed-401f2d8eb402 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26619" uid: 641d2e0c-2399-4bdb-a3d9-24baf08bf0a6 kind: Event lastTimestamp: "2026-07-28T17:49:19Z" message: InferenceService [error-404-isvc-c0ed1] is Ready metadata: creationTimestamp: "2026-07-28T17:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:19Z" name: error-404-isvc-c0ed1.18c6856259f3bce7 namespace: kserve-ci-e2e-test resourceVersion: "26620" uid: a3781640-86e0-4169-8502-c19ea297e17a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:38:11.163391Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20660" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c57a1-predictor-9bd97b66d-twn7s to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6ae679cc7 namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 1bf88e6d-8a33-4277-9af8-5bb90217be31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6c889be3b namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 35241caf-e685-47c9-83d0-eedc7b135ef5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6ca5b4de1 namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: 76abb1b9-e5a7-46df-ad2a-e2baa6c71f77 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6ccacbcfe namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: a2f16e38-94dc-4d91-aff0-8286d3e4a644 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6cd1f5d1d namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 1e22a607-1cb8-4b0a-8ec6-f54d6445df80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6cd296fea namespace: kserve-ci-e2e-test resourceVersion: "20700" uid: 09d5f1ba-e9bb-4361-8cb9-8c9028c58d18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6d0131482 namespace: kserve-ci-e2e-test resourceVersion: "20702" uid: 2af33afd-976c-4a94-b071-38af47d6bef9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6d1118e68 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 7db3b547-35d1-478c-9782-907c8aeffa25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:47:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:38Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c684c6ef9e8855 namespace: kserve-ci-e2e-test resourceVersion: "25157" uid: 90c1b681-ff63-48a2-b18d-ea99ccda445e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c6854a35d2de82 namespace: kserve-ci-e2e-test resourceVersion: "25077" uid: b725833a-2fe1-4268-8fa0-2881f2b3c35f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c6854a35d35956 namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 4afaf86d-655f-45ed-aefc-943b1e698f06 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s namespace: kserve-ci-e2e-test resourceVersion: "20662" uid: 45c665a0-4abc-4471-905e-b157af3578c5 kind: Event lastTimestamp: "2026-07-28T17:47:38Z" message: 'Readiness probe failed: Get "https://10.132.0.33:8643/healthz": dial tcp 10.132.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:38Z" name: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s.18c6854ab80f4879 namespace: kserve-ci-e2e-test resourceVersion: "25154" uid: 97200a1b-3f9b-4014-aea2-4cd4c766eb4b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c57a1-predictor-9bd97b66d namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 10f9d8e0-4195-4431-b70f-51103595e501 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created pod: error-404-isvc-c57a1-predictor-9bd97b66d-twn7s' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor-9bd97b66d.18c684c6adf7f846 namespace: kserve-ci-e2e-test resourceVersion: "20661" uid: 16916c09-051d-4f8b-85ec-df58f3f0b330 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: d1b54771-7f84-4edc-a2ff-bd143ef4a7dd kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Scaled up replica set error-404-isvc-c57a1-predictor-9bd97b66d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1-predictor.18c684c6ac5521f9 namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 12098ae2-caa7-4e73-a667-f88514670607 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 862ae297-0143-4f10-9c73-237c0efa0304 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: error-404-isvc-c57a1-predictor.18c684ca2aa29487 namespace: kserve-ci-e2e-test resourceVersion: "21103" uid: 4106c2b6-f3f8-4a72-b1b8-7c1ed608b1d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 862ae297-0143-4f10-9c73-237c0efa0304 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: error-404-isvc-c57a1-predictor.18c684ca2b105eb9 namespace: kserve-ci-e2e-test resourceVersion: "21104" uid: ae45a510-c268-472e-9bce-0885cf156182 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:38:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 862ae297-0143-4f10-9c73-237c0efa0304 kind: Event lastTimestamp: "2026-07-28T17:39:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:41Z" name: error-404-isvc-c57a1-predictor.18c684cda9c5d8fc namespace: kserve-ci-e2e-test resourceVersion: "21705" uid: ad95ae19-166d-45b2-a2d2-bb37a6cfea96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:38:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20842" uid: 862ae297-0143-4f10-9c73-237c0efa0304 kind: Event lastTimestamp: "2026-07-28T17:39:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:41Z" name: error-404-isvc-c57a1-predictor.18c684cdaa2d6cb3 namespace: kserve-ci-e2e-test resourceVersion: "21706" uid: 2cd7f7c8-2c7d-4c48-b679-0f908c52ecd0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 2972888c-1a4b-42f6-a1b0-b98a54e60e6f kind: Event lastTimestamp: "2026-07-28T17:38:58Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:58Z" name: error-404-isvc-c57a1.18c684c6ac1f0973 namespace: kserve-ci-e2e-test resourceVersion: "21169" uid: 25fca14e-100a-423f-a381-f916a777c01c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 2972888c-1a4b-42f6-a1b0-b98a54e60e6f kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Failed to update status for InferenceService "error-404-isvc-c57a1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c57a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1.18c684c6ac785004 namespace: kserve-ci-e2e-test resourceVersion: "20657" uid: 035e0c69-1ea8-46bd-ae15-df7298be578b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20638" uid: 2972888c-1a4b-42f6-a1b0-b98a54e60e6f kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c57a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:11Z" name: error-404-isvc-c57a1.18c684c6ac78a413 namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 4b12481a-409a-4013-9066-ef5bacf6230e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21166" uid: 2972888c-1a4b-42f6-a1b0-b98a54e60e6f kind: Event lastTimestamp: "2026-07-28T17:38:58Z" message: InferenceService [error-404-isvc-c57a1] is Ready metadata: creationTimestamp: "2026-07-28T17:38:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:58Z" name: error-404-isvc-c57a1.18c684d1ab3fe97f namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 1d6f1a9d-deb9-417f-bd90-b6679e583e8c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:39:07.155744Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21324" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c7add-predictor-6d685ddd8-cd8kt to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3b7cf1757 namespace: kserve-ci-e2e-test resourceVersion: "21328" uid: 2484937c-d2fc-4595-b6a1-9fc5b70a9f0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c7add-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3c3ad1ae9 namespace: kserve-ci-e2e-test resourceVersion: "21342" uid: 7b00341f-19a0-4162-a2ba-eb8b6698ce4a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3f51184b2 namespace: kserve-ci-e2e-test resourceVersion: "21360" uid: 70adcbbb-96c5-40b7-a254-15fd114be6fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3f65d6955 namespace: kserve-ci-e2e-test resourceVersion: "21362" uid: 8c56f20f-3c1e-46d2-b918-16f134074c15 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3f949b348 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: b2b7e799-8aa3-4481-8dee-c2370ac0e694 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3f9d0d981 namespace: kserve-ci-e2e-test resourceVersion: "21364" uid: 3c60820b-9489-463a-8a79-d00b86a5fc6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3f9e9f1f3 namespace: kserve-ci-e2e-test resourceVersion: "21365" uid: 2f528536-66d4-4d56-a3dc-58f5da62ccbc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3fe44017e namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 0bc99037-beb5-45e1-a2a1-50d28423de37 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:08Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d3fec13cc0 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 04fb667b-c50f-4721-bbd4-6a4fcba4800b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:48:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:24Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c684d40856c5ec namespace: kserve-ci-e2e-test resourceVersion: "25887" uid: 279e8d82-1dd0-43fc-9519-9592d43a04b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c68554dd576468 namespace: kserve-ci-e2e-test resourceVersion: "25702" uid: ccf44505-2a23-4611-aa07-6e5a80e6b7e8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:21Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c68554dd5d0624 namespace: kserve-ci-e2e-test resourceVersion: "25707" uid: 2073c856-896d-447e-bc66-e436b966de9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt namespace: kserve-ci-e2e-test resourceVersion: "21326" uid: d8a795a5-5625-4829-8306-0fc96a2377dd kind: Event lastTimestamp: "2026-07-28T17:48:24Z" message: 'Readiness probe failed: Get "https://10.132.0.35:8643/healthz": dial tcp 10.132.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:24Z" name: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt.18c685557cbd7225 namespace: kserve-ci-e2e-test resourceVersion: "25885" uid: 8a1fa8f6-f4da-40c2-82bd-bde18c06c7b6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c7add-predictor-6d685ddd8 namespace: kserve-ci-e2e-test resourceVersion: "21317" uid: 40e66a90-09a1-4ac4-a2f5-75ef67fccaeb kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Created pod: error-404-isvc-c7add-predictor-6d685ddd8-cd8kt' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add-predictor-6d685ddd8.18c684d3b763514b namespace: kserve-ci-e2e-test resourceVersion: "21325" uid: 32dc548f-73ef-4ddf-a548-06e44a06cfb4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21315" uid: aa5664e5-3d7d-4284-80f2-cafff294da5c kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Scaled up replica set error-404-isvc-c7add-predictor-6d685ddd8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add-predictor.18c684d3b5d11d5a namespace: kserve-ci-e2e-test resourceVersion: "21318" uid: 768c2418-a485-4780-a132-a658778ab1c7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: c2dbf035-aa23-4424-9e38-8164d0bfd227 kind: Event lastTimestamp: "2026-07-28T17:39:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:22Z" name: error-404-isvc-c7add-predictor.18c684d7346a6354 namespace: kserve-ci-e2e-test resourceVersion: "21546" uid: 0ff8e3a4-78a2-49d7-a332-619904afae84 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21316" uid: c2dbf035-aa23-4424-9e38-8164d0bfd227 kind: Event lastTimestamp: "2026-07-28T17:39:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:22Z" name: error-404-isvc-c7add-predictor.18c684d734dff5f6 namespace: kserve-ci-e2e-test resourceVersion: "21548" uid: c8fc5185-93fd-4bd7-b54c-cc9df36f3ee1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:39:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: c2dbf035-aa23-4424-9e38-8164d0bfd227 kind: Event lastTimestamp: "2026-07-28T17:40:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:22Z" name: error-404-isvc-c7add-predictor.18c684dab38eedf4 namespace: kserve-ci-e2e-test resourceVersion: "22070" uid: 99fcb86c-7e94-4788-a401-75c46cd8939d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:39:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: c2dbf035-aa23-4424-9e38-8164d0bfd227 kind: Event lastTimestamp: "2026-07-28T17:40:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:22Z" name: error-404-isvc-c7add-predictor.18c684dab3e18904 namespace: kserve-ci-e2e-test resourceVersion: "22071" uid: 24d47ba4-5f3d-4765-b8a2-e9c2137d135e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 394d5d4a-80d9-4355-a04a-24fcfcc9bc3e kind: Event lastTimestamp: "2026-07-28T17:39:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:54Z" name: error-404-isvc-c7add.18c684d3b5e50bb1 namespace: kserve-ci-e2e-test resourceVersion: "21816" uid: cca6d81b-c6ce-47b9-9177-fd8a7cfc95a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 394d5d4a-80d9-4355-a04a-24fcfcc9bc3e kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Failed to update status for InferenceService "error-404-isvc-c7add": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c7add": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add.18c684d3b6435083 namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: 6e51ae21-c035-4622-8a4d-89e87fa2049e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21297" uid: 394d5d4a-80d9-4355-a04a-24fcfcc9bc3e kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c7add": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:07Z" name: error-404-isvc-c7add.18c684d3b643929a namespace: kserve-ci-e2e-test resourceVersion: "21323" uid: 254d1f91-aa02-4f2a-97d1-aa0f9fbb616d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21814" uid: 394d5d4a-80d9-4355-a04a-24fcfcc9bc3e kind: Event lastTimestamp: "2026-07-28T17:39:54Z" message: InferenceService [error-404-isvc-c7add] is Ready metadata: creationTimestamp: "2026-07-28T17:39:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:54Z" name: error-404-isvc-c7add.18c684dec3491144 namespace: kserve-ci-e2e-test resourceVersion: "21815" uid: 95dd23c3-b3b8-4a09-a0f8-65e6d1a2a011 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:26:11.962519Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14315" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-cc402-predictor-696c97fc9d-gjdch to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:26:11Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6841f3ab134b5 namespace: kserve-ci-e2e-test resourceVersion: "14320" uid: c45e703a-7991-4e3d-a75e-f4be27ad94f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-cc402-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6841f467f48a3 namespace: kserve-ci-e2e-test resourceVersion: "14357" uid: 4cb91cce-def4-42d3-8207-a65951fca625 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:26:13Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6841f7c3f29d0 namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 75cd574d-966e-421f-8e8b-73eea28a25c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:13Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6841f7dc8e6eb namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: c7d175f2-14f5-4ce6-8eb2-f0bafa2fc652 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:27Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1814" in 14.904s (14.904s including waiting). Image size: 1333938285 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:27Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68422f6276e5a namespace: kserve-ci-e2e-test resourceVersion: "14556" uid: 7f3c4b68-86a6-4986-b8b2-d8797aa5def8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68422fa4fa649 namespace: kserve-ci-e2e-test resourceVersion: "14559" uid: 8018c455-4ac9-4484-9564-17fd4ed566f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68422fab6f84e namespace: kserve-ci-e2e-test resourceVersion: "14561" uid: 8a3034ac-b3a1-4416-9472-93fba4fedea4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68422fac5341c namespace: kserve-ci-e2e-test resourceVersion: "14563" uid: 7137af47-3a3d-478c-a250-c80b027766a1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.596s (2.596s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68423958a66b7 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: bb7e595d-cbbe-4594-a601-1dcdf0ffff0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6842398beb08e namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: eed31176-5f50-4a98-a0c5-678b55b3276f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c684239926dc8c namespace: kserve-ci-e2e-test resourceVersion: "14584" uid: cb9be889-6f65-48f9-b11a-266fa0f52d83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c68423c8977686 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: e1a9f14b-4ba8-47cf-bd24-635346a3258d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6843519e78492 namespace: kserve-ci-e2e-test resourceVersion: "15355" uid: 343e7d09-e176-4fe5-9231-589684e9bc38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:45Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c6843519e81152 namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 4b1e0902-cd99-40d3-80f6-82dd6d76c5c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch namespace: kserve-ci-e2e-test resourceVersion: "14317" uid: f728a695-7eb1-4d79-94dc-6b7870f877d9 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Readiness probe failed: Get "https://10.132.0.23:8643/healthz": dial tcp 10.132.0.23:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: error-404-isvc-cc402-predictor-696c97fc9d-gjdch.18c684357ad43e34 namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: b6b80b90-d4bb-4f63-a07c-285a9f75dedf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-cc402-predictor-696c97fc9d namespace: kserve-ci-e2e-test resourceVersion: "14307" uid: bc943335-106e-444f-a97e-15bc9246f7ef kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: 'Created pod: error-404-isvc-cc402-predictor-696c97fc9d-gjdch' metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: error-404-isvc-cc402-predictor-696c97fc9d.18c6841f3a3e3310 namespace: kserve-ci-e2e-test resourceVersion: "14316" uid: c17c038c-4109-471f-b069-c9c6281552ed reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14304" uid: d7e65868-ef6e-4f7e-8a26-607bddb4efc0 kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: Scaled up replica set error-404-isvc-cc402-predictor-696c97fc9d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: error-404-isvc-cc402-predictor.18c6841f38acfd3a namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: 07015eb5-3f54-40cb-b70c-e5775f3cb037 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 3207b406-5eba-432e-a13e-953e3a78ec80 kind: Event lastTimestamp: "2026-07-28T17:27:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:11Z" name: error-404-isvc-cc402-predictor.18c68422b734d979 namespace: kserve-ci-e2e-test resourceVersion: "14951" uid: 6f078de6-7e8b-49a4-b9aa-1708fc59f04c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: 3207b406-5eba-432e-a13e-953e3a78ec80 kind: Event lastTimestamp: "2026-07-28T17:27:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:11Z" name: error-404-isvc-cc402-predictor.18c68422b7880177 namespace: kserve-ci-e2e-test resourceVersion: "14953" uid: 38e7e9be-8cfd-44f7-970b-584f400cf5a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 3207b406-5eba-432e-a13e-953e3a78ec80 kind: Event lastTimestamp: "2026-07-28T17:27:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:42Z" name: error-404-isvc-cc402-predictor.18c68430b28ae460 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: e6401164-359f-4c33-bd14-871b735b84b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:27:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14544" uid: 3207b406-5eba-432e-a13e-953e3a78ec80 kind: Event lastTimestamp: "2026-07-28T17:27:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:42Z" name: error-404-isvc-cc402-predictor.18c68430b31c0d48 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 734bf2cf-7d3b-4bc6-b40f-5024abf92238 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc402 namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: de9ed007-5479-40d6-89a1-63638d6ffff0 kind: Event lastTimestamp: "2026-07-28T17:27:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:17Z" name: error-404-isvc-cc402.18c6841f38acbb5c namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 0e367d7e-965c-424f-8325-ca0cdacd6ae4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-cc402 namespace: kserve-ci-e2e-test resourceVersion: "15017" uid: de9ed007-5479-40d6-89a1-63638d6ffff0 kind: Event lastTimestamp: "2026-07-28T17:27:17Z" message: InferenceService [error-404-isvc-cc402] is Ready metadata: creationTimestamp: "2026-07-28T17:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:17Z" name: error-404-isvc-cc402.18c6842e849ac9e0 namespace: kserve-ci-e2e-test resourceVersion: "15018" uid: 094a697a-f1d5-4d34-92ec-300780a92fff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:26:11.663652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14234" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr to ip-10-0-129-206.ec2.internal metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:26:11Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6841f28e0e189 namespace: kserve-ci-e2e-test resourceVersion: "14240" uid: 5c308736-f13c-4f4d-81ee-40df759315ef reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14243" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6841f4272110e namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 72b2916f-dd9c-484f-88ba-8a3ecf9bc971 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" already present on machine metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6841f440fa636 namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 18fa5f05-008a-4980-ae53-e2bdbabb7440 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6841f47582e46 namespace: kserve-ci-e2e-test resourceVersion: "14358" uid: cf67f159-94be-473d-9594-29af6120cfd5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6841f47bc7c68 namespace: kserve-ci-e2e-test resourceVersion: "14359" uid: 073a435e-492e-4455-a883-066a1b50046c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:16Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T17:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:16Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6842052e12cae namespace: kserve-ci-e2e-test resourceVersion: "14447" uid: ae566f5f-fe03-4350-abe6-51eea58c7603 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:43Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 27.118s (27.118s including waiting). Image size: 1555877302 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:43Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68426a348138f namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: 4f0695bb-38b5-41a5-ba2d-8a84a22581a0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:43Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68426a9ac896f namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 86fcbf43-f0f4-4132-a789-c3ae5effca5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:26:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:43Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68426ab4e389c namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: 15c1a5da-1fe3-43ac-9f8c-93add72663a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68426b4842ff9 namespace: kserve-ci-e2e-test resourceVersion: "14703" uid: 1f4afeaf-9f8f-490d-a675-82c623d129df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.072s (3.072s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c684276bac1038 namespace: kserve-ci-e2e-test resourceVersion: "14735" uid: 18c72389-a1a2-42a5-91b1-309d80b1f92f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c684276f207308 namespace: kserve-ci-e2e-test resourceVersion: "14738" uid: baf995ca-ce7f-4630-9eb1-1baf34899239 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c684276fb68505 namespace: kserve-ci-e2e-test resourceVersion: "14739" uid: 5d4c01e2-2daf-48f9-bcc6-8282b7bbc0e5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:28:03Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:03Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c6842798778b3f namespace: kserve-ci-e2e-test resourceVersion: "15594" uid: 6a1930c9-6b72-4fb1-aa21-005284cda39a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68442245d179c namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 433f57c3-fde8-452e-826a-31ab575da27b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c68442245d9273 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: ba9633c2-dd1d-442d-a3db-423f37f94427 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr namespace: kserve-ci-e2e-test resourceVersion: "14236" uid: 149c3c47-5c42-4518-a006-4977e053d6f1 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Readiness probe failed: Get "https://10.134.0.27:8643/healthz": dial tcp 10.134.0.27:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr.18c684429ae18b1d namespace: kserve-ci-e2e-test resourceVersion: "16128" uid: 0cf080c8-113d-4e67-ab3a-d368da6af5fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-6d55f5d589 namespace: kserve-ci-e2e-test resourceVersion: "14222" uid: f6874ae4-5c3c-4013-a86c-931fcec1dd2b kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-6d55f5d589-c5gzr' metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: isvc-sklearn-graph-1-predictor-6d55f5d589.18c6841f2865a85d namespace: kserve-ci-e2e-test resourceVersion: "14235" uid: 92a73ec5-2575-40ad-9c2d-1be5890ff922 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14221" uid: f45e20fe-a099-4c99-96f8-e66ee64f62fd kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-6d55f5d589 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: isvc-sklearn-graph-1-predictor.18c6841f21043072 namespace: kserve-ci-e2e-test resourceVersion: "14224" uid: 4264220b-387b-412e-aa1c-4b81d12b5009 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: a522a7fc-7f49-4d24-8f36-229e8be12bfd kind: Event lastTimestamp: "2026-07-28T17:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:26Z" name: isvc-sklearn-graph-1-predictor.18c68422a6c2fe20 namespace: kserve-ci-e2e-test resourceVersion: "15085" uid: d941e945-8483-44a0-8f4f-b5b77fb6a1cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14229" uid: a522a7fc-7f49-4d24-8f36-229e8be12bfd kind: Event lastTimestamp: "2026-07-28T17:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:26Z" name: isvc-sklearn-graph-1-predictor.18c68422a733cf57 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 9deb5742-1481-477f-aae1-55b969799e58 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: a522a7fc-7f49-4d24-8f36-229e8be12bfd kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-1-predictor.18c6842d246463aa namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: 34cb728a-848c-40ba-a67c-e3d11ae9ae93 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: a522a7fc-7f49-4d24-8f36-229e8be12bfd kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-1-predictor.18c6842d24d81404 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 01a83791-7b40-4036-bf2f-4fc34893c711 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14211" uid: 97524dc5-bf00-4b1d-afd4-47a1c18b2440 kind: Event lastTimestamp: "2026-07-28T17:28:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:09Z" name: isvc-sklearn-graph-1.18c6841f271b96bc namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 190415ab-00f3-4d09-af33-2156c5692531 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: 97524dc5-bf00-4b1d-afd4-47a1c18b2440 kind: Event lastTimestamp: "2026-07-28T17:28:09Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T17:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:09Z" name: isvc-sklearn-graph-1.18c6843a97b59c15 namespace: kserve-ci-e2e-test resourceVersion: "15652" uid: e6ab9073-e043-4c57-8b84-7360921487c5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:26:12.082381Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14341" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7d44765f95-b55fd to ip-10-0-129-206.ec2.internal metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f41d62a35 namespace: kserve-ci-e2e-test resourceVersion: "14345" uid: 832c014f-33e9-4cf8-894b-e75769137d0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f4d6c13d9 namespace: kserve-ci-e2e-test resourceVersion: "14374" uid: f31b6d13-512c-4779-a8dc-2fac43d54970 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:26:13Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f7ee10d6a namespace: kserve-ci-e2e-test resourceVersion: "14410" uid: 76662fdb-5ede-4413-86b9-2d46ff46a533 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" already present on machine metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:13Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f8085687a namespace: kserve-ci-e2e-test resourceVersion: "14412" uid: ccba05de-a7dc-4e52-92cb-40172b26c830 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:13Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f837e0c76 namespace: kserve-ci-e2e-test resourceVersion: "14415" uid: 9ed14ec5-2a54-4253-9115-76d5e4e1b618 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:13Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:26:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:13Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6841f83dbf343 namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: b41ad12b-2b62-4fcc-b11a-0fc1f770ea13 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:17Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T17:26:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:17Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c684208f75eee7 namespace: kserve-ci-e2e-test resourceVersion: "14463" uid: 1fe70124-73ee-46e6-9298-cd83e195c738 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1814" in 26.479s (26.479s including waiting). Image size: 1555877302 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c68426b9c80bb3 namespace: kserve-ci-e2e-test resourceVersion: "14705" uid: ef39bb8f-3f1e-4818-89ca-2aa1ab2fe564 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c68426c035b734 namespace: kserve-ci-e2e-test resourceVersion: "14707" uid: f248df62-1711-4b78-b2d5-cbcdbea765fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c68426c12deb7e namespace: kserve-ci-e2e-test resourceVersion: "14710" uid: 2d4debf4-008a-42fa-8233-58ddf3be9253 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c68426c13b73ea namespace: kserve-ci-e2e-test resourceVersion: "14711" uid: 7e8ead2e-47eb-45a5-9dfd-81c55f99d679 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.917s (2.917s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c684276f201661 namespace: kserve-ci-e2e-test resourceVersion: "14737" uid: 81d21aad-6100-4fc4-a9e0-8dbcb324627a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6842773cb8cd9 namespace: kserve-ci-e2e-test resourceVersion: "14741" uid: 897b3678-6377-469a-9bcb-f28470bd8791 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c684277481b7fc namespace: kserve-ci-e2e-test resourceVersion: "14745" uid: 447ed7a2-c030-4458-bd0f-f5e88c8ac01a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:27:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:59Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c684279856a33f namespace: kserve-ci-e2e-test resourceVersion: "15554" uid: 75f24d3a-cb22-4dab-98b7-9c36e5515fdf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c68442209fb2d0 namespace: kserve-ci-e2e-test resourceVersion: "16029" uid: 3670ceb4-82b8-4234-b7bb-2a0a7c803e98 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c6844220a02d49 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 9607c79a-8a13-4bed-8297-2b17124743e9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd namespace: kserve-ci-e2e-test resourceVersion: "14343" uid: 4ae5b6aa-0fb0-4cfa-ba5c-838123a8d2ff kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd.18c684429ae228bb namespace: kserve-ci-e2e-test resourceVersion: "16129" uid: 1f40cde0-e886-4f9d-b09d-2954f12889da reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7d44765f95 namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 9132776e-1959-4d61-89e3-489a4775fa25 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7d44765f95-b55fd' metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-2-predictor-7d44765f95.18c6841f413ee100 namespace: kserve-ci-e2e-test resourceVersion: "14342" uid: b8ccfbb6-b00b-4bee-94ee-f51313274715 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: e26c6581-c154-4153-a749-922aeb832b7f kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7d44765f95 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:12Z" name: isvc-sklearn-graph-2-predictor.18c6841f3ea2554e namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 19fce363-af4f-4806-837e-2e0802079b15 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:26:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 010aa781-d881-4833-ad0a-f1dfb7a40b88 kind: Event lastTimestamp: "2026-07-28T17:26:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:57Z" name: isvc-sklearn-graph-2-predictor.18c68422bd4f5eab namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 97a59ba8-3f97-4752-b182-2113647d25d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:26:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 010aa781-d881-4833-ad0a-f1dfb7a40b88 kind: Event lastTimestamp: "2026-07-28T17:26:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:57Z" name: isvc-sklearn-graph-2-predictor.18c68422bdd5d0bd namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 4cd7bde0-fa38-4faf-acfb-27ab3b6c5115 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 010aa781-d881-4833-ad0a-f1dfb7a40b88 kind: Event lastTimestamp: "2026-07-28T17:28:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:27Z" name: isvc-sklearn-graph-2-predictor.18c6842d3a3ceecd namespace: kserve-ci-e2e-test resourceVersion: "15793" uid: 6b04d39a-b91c-4439-be37-275f2b5ab7f8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14548" uid: 010aa781-d881-4833-ad0a-f1dfb7a40b88 kind: Event lastTimestamp: "2026-07-28T17:28:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:27Z" name: isvc-sklearn-graph-2-predictor.18c6842d3ab389d8 namespace: kserve-ci-e2e-test resourceVersion: "15794" uid: f42a8658-ca1c-47fa-b2bf-ffc0b852735b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14319" uid: 4df2304c-428f-4ff8-8130-23bf2b645555 kind: Event lastTimestamp: "2026-07-28T17:28:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:09Z" name: isvc-sklearn-graph-2.18c6841f3e91339e namespace: kserve-ci-e2e-test resourceVersion: "15653" uid: a261216e-0c9d-43ec-be86-0720115f9ab4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4df2304c-428f-4ff8-8130-23bf2b645555 kind: Event lastTimestamp: "2026-07-28T17:28:09Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T17:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:09Z" name: isvc-sklearn-graph-2.18c6843a949344b1 namespace: kserve-ci-e2e-test resourceVersion: "15649" uid: 071e19b3-fefb-4e63-bb05-9f0399cd412f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:26:11.885278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-8fhcb to ip-10-0-129-206.ec2.internal metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:26:11Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6841f36169b24 namespace: kserve-ci-e2e-test resourceVersion: "14293" uid: 90d205f7-657a-4344-b9bb-c4f4134575fe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14303" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:26:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6841f4ff81a92 namespace: kserve-ci-e2e-test resourceVersion: "14378" uid: 7a57f18a-92b3-46f7-936a-1e7bc19cc168 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:824dd4f70c10bfd12c956cfe8488915467e61b05d1a785bf64aa919cb1efc5da" already present on machine metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6841f5186b187 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 88b096ee-1629-41a7-9971-044b4ed0966c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6841f5450553e namespace: kserve-ci-e2e-test resourceVersion: "14385" uid: 334ffe2a-514c-4948-944f-77fadc279e63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6841f54f4fab5 namespace: kserve-ci-e2e-test resourceVersion: "14387" uid: e452115f-21ab-4293-ac8c-50ef0f26cd2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:16Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T17:26:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:16Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6842052f872cc namespace: kserve-ci-e2e-test resourceVersion: "14448" uid: b305ac27-07a6-4405-993c-a28aa9d49bba reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 27.478s (27.478s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c68426b8cd6d72 namespace: kserve-ci-e2e-test resourceVersion: "14704" uid: 314703ef-4b41-4a63-bb90-2df454bd6a3d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c68426bfe02f00 namespace: kserve-ci-e2e-test resourceVersion: "14706" uid: 5c645641-26e6-45f4-aafd-99b9f6156aa9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c68426c0f2fc12 namespace: kserve-ci-e2e-test resourceVersion: "14708" uid: 217625d1-9825-4b00-b4ab-773d66434ca9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:44Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:26:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:44Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c68426c1061026 namespace: kserve-ci-e2e-test resourceVersion: "14709" uid: f08e6e73-2ac0-4f24-8978-34d3c34e04af reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.934s (2.934s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c684276ff1c459 namespace: kserve-ci-e2e-test resourceVersion: "14740" uid: 43a633ac-3787-4293-aec3-f30929c4d3d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6842773d5b327 namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: b517000e-9255-418b-b432-84f07ca6fb58 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:26:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:47Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c6842774732b49 namespace: kserve-ci-e2e-test resourceVersion: "14744" uid: 0ce236dc-36a3-44e0-8b22-b9f5094717b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:26:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:26:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c68427983887f7 namespace: kserve-ci-e2e-test resourceVersion: "16130" uid: 0aaa5288-088c-4d26-a572-ed81b80c2533 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c684421ce5c733 namespace: kserve-ci-e2e-test resourceVersion: "15999" uid: d69021c4-d1e9-4c0c-ae97-6c797f239e7f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c684421ce64e2e namespace: kserve-ci-e2e-test resourceVersion: "16001" uid: 7ac0712f-9ae5-430a-ac47-28c5fafa88b4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb namespace: kserve-ci-e2e-test resourceVersion: "14292" uid: c37a984c-db24-4b00-aa53-82cfe0722356 kind: Event lastTimestamp: "2026-07-28T17:28:43Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-8fhcb.18c684429ae02787 namespace: kserve-ci-e2e-test resourceVersion: "16127" uid: 49ecff7a-13de-4653-b32e-6b69e41a31a2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-206.ec2.internal source: component: kubelet host: ip-10-0-129-206.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: caa22fb9-3711-4eb9-b303-6ad77bc799f8 kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-8fhcb' metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c6841f3588800f namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 850aa262-627e-4b67-a2c1-237ee62a44f6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14274" uid: 14adfc9e-e8ae-4864-9f1e-48ddaa201066 kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: isvc-xgboost-graph-predictor.18c6841f32dcab54 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: f8dd561e-595d-470f-aa8b-83b14a57a8f1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: dd014e0f-3a3d-4446-8b33-90e0ea4a5cf0 kind: Event lastTimestamp: "2026-07-28T17:27:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:26Z" name: isvc-xgboost-graph-predictor.18c68422b18cc64c namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: c729bf2e-65fe-4cac-8da7-64d9712c53a7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14277" uid: dd014e0f-3a3d-4446-8b33-90e0ea4a5cf0 kind: Event lastTimestamp: "2026-07-28T17:27:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:26Z" name: isvc-xgboost-graph-predictor.18c68422b1f69920 namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: 109c8ea3-01c7-4c5b-9d07-0cfba89e4511 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: dd014e0f-3a3d-4446-8b33-90e0ea4a5cf0 kind: Event lastTimestamp: "2026-07-28T17:28:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:26Z" name: isvc-xgboost-graph-predictor.18c684342bb91f4d namespace: kserve-ci-e2e-test resourceVersion: "15790" uid: e63e2913-787c-4b74-86fc-e777e0b11de2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:27:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14541" uid: dd014e0f-3a3d-4446-8b33-90e0ea4a5cf0 kind: Event lastTimestamp: "2026-07-28T17:28:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:27:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:26Z" name: isvc-xgboost-graph-predictor.18c684342c27e65d namespace: kserve-ci-e2e-test resourceVersion: "15791" uid: 889f924f-27a7-4047-a401-ba4eea90e241 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14263" uid: e72ed6e6-dbdf-4e14-a7bb-4210af1503bf kind: Event lastTimestamp: "2026-07-28T17:27:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:54Z" name: isvc-xgboost-graph.18c6841f33114bfc namespace: kserve-ci-e2e-test resourceVersion: "15505" uid: 3212403e-288a-4be1-93e6-3b6cd22da5fe reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: e72ed6e6-dbdf-4e14-a7bb-4210af1503bf kind: Event lastTimestamp: "2026-07-28T17:27:54Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T17:27:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:54Z" name: isvc-xgboost-graph.18c68436fd5115d6 namespace: kserve-ci-e2e-test resourceVersion: "15503" uid: f48f1731-bcb6-4f9a-b8c5-90ca3150bdd1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:28:31.542441Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-c5b4df566-hwmdd to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer-c5b4df566-hwmdd.18c6843fba4dcdb8 namespace: kserve-ci-e2e-test resourceVersion: "15850" uid: c70e3431-a565-416f-bcac-9d2a852e5d26 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15853" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:28:31Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer-c5b4df566-hwmdd.18c6843fd3c0845a namespace: kserve-ci-e2e-test resourceVersion: "15865" uid: 5d435549-c04f-4406-b965-a97be85add66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:28:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer-c5b4df566-hwmdd.18c6843fd52348d1 namespace: kserve-ci-e2e-test resourceVersion: "15868" uid: d47e56db-53ae-4bc4-8274-2b7d04244414 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:28:32Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T17:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:32Z" name: model-chainer-c5b4df566-hwmdd.18c6843fd80cabe1 namespace: kserve-ci-e2e-test resourceVersion: "15869" uid: 293ac177-866b-4eef-a503-432b0d04d765 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:28:32Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T17:28:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:32Z" name: model-chainer-c5b4df566-hwmdd.18c6843fd8925a18 namespace: kserve-ci-e2e-test resourceVersion: "15870" uid: a76d741b-2f49-435f-af42-44c9f623fd6e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: model-chainer-c5b4df566-hwmdd.18c6844215593ca5 namespace: kserve-ci-e2e-test resourceVersion: "15975" uid: e19b12e3-9319-4bac-ac57-620ed00c8054 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:28:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-c5b4df566-hwmdd namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: d554db6a-5bc5-474f-b208-cf63d59d55bc kind: Event lastTimestamp: "2026-07-28T17:29:09Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:28:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:09Z" name: model-chainer-c5b4df566-hwmdd.18c68442a2a19075 namespace: kserve-ci-e2e-test resourceVersion: "16388" uid: d7d295d2-e3a6-45f2-b886-6053665f7099 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-c5b4df566 namespace: kserve-ci-e2e-test resourceVersion: "15842" uid: 046d4c70-d8ad-47c5-971d-36d638eed00f kind: Event lastTimestamp: "2026-07-28T17:28:31Z" message: 'Created pod: model-chainer-c5b4df566-hwmdd' metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer-c5b4df566.18c6843fba174dd5 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: 77f6b792-2c98-42e1-b765-255387433308 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15840" uid: 07f3691f-4e19-4f27-a1cc-fe8d24841406 kind: Event lastTimestamp: "2026-07-28T17:28:31Z" message: Scaled up replica set model-chainer-c5b4df566 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer.18c6843fb90058a1 namespace: kserve-ci-e2e-test resourceVersion: "15843" uid: 079799d1-cbca-4250-b800-55f9eb22c87a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: 1ce2542a-09d4-4e89-8719-bc065c72350b kind: Event lastTimestamp: "2026-07-28T17:28:31Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T17:28:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:31Z" name: model-chainer.18c6843fbc9f63e4 namespace: kserve-ci-e2e-test resourceVersion: "15859" uid: 0170a071-d6be-4347-9062-4e6f3804389e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:38:56.782708Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21118" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-4108d-594769977b-sx7fn to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:38:56Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d14d871d72 namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 8b6f829f-10b7-48ec-9593-94a22c635897 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:38:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-4108d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:38:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:57Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d15e4ed9c2 namespace: kserve-ci-e2e-test resourceVersion: "21140" uid: 5e9c5de6-0a46-4076-a64d-3b2b6ed15889 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:57Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21130" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:38:57Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:38:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:38:57Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d18add0046 namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 6992e5da-cf1e-42fd-aa20-0511a5a4d9a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4108d} kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:38:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:57Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d18c5238ae namespace: kserve-ci-e2e-test resourceVersion: "21151" uid: 339212a2-34ae-4b5c-9e0f-dfc977fd7dae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4108d} kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:38:57Z" message: 'Created container: sequence-graph-4108d' metadata: creationTimestamp: "2026-07-28T17:38:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:57Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d18f132b50 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 39a21567-3a0f-4852-a51f-5bce454c2f07 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4108d} kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:38:57Z" message: Started container sequence-graph-4108d metadata: creationTimestamp: "2026-07-28T17:38:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:57Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d18f81b896 namespace: kserve-ci-e2e-test resourceVersion: "21155" uid: 918ff367-f720-40db-96c9-699d1a713830 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4108d} kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:39:06Z" message: Stopping container sequence-graph-4108d metadata: creationTimestamp: "2026-07-28T17:39:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:06Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d3a856df92 namespace: kserve-ci-e2e-test resourceVersion: "21245" uid: b3d7addc-bbdc-4568-8c36-b0e18be96f8b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4108d} kind: Pod name: sequence-graph-4108d-594769977b-sx7fn namespace: kserve-ci-e2e-test resourceVersion: "21120" uid: 65a3f603-5eb5-442c-a410-893923546566 kind: Event lastTimestamp: "2026-07-28T17:39:34Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:34Z" name: sequence-graph-4108d-594769977b-sx7fn.18c684d44ecc3eb0 namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 0a2d6b93-8d82-4199-b7d4-fc09174cd238 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-4108d-594769977b namespace: kserve-ci-e2e-test resourceVersion: "21115" uid: 1c9e5aa5-9225-4315-9f49-4e68a11571bf kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'Created pod: sequence-graph-4108d-594769977b-sx7fn' metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: sequence-graph-4108d-594769977b.18c684d14d13b9f5 namespace: kserve-ci-e2e-test resourceVersion: "21119" uid: 75301cf1-ca6d-4bca-b322-cae505909133 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-4108d namespace: kserve-ci-e2e-test resourceVersion: "21113" uid: fe98f3a2-7a90-47c9-bb80-b68179a28e06 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: Scaled up replica set sequence-graph-4108d-594769977b from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: sequence-graph-4108d.18c684d14c2bc49b namespace: kserve-ci-e2e-test resourceVersion: "21116" uid: 2e2cc4fc-0786-470c-ae7c-e8993dd99a61 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-4108d namespace: kserve-ci-e2e-test resourceVersion: "21133" uid: c435af52-9fda-4c00-b728-02c0ddc8dee5 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: InferenceGraph [sequence-graph-4108d] is Ready metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:56Z" name: sequence-graph-4108d.18c684d151d8147e namespace: kserve-ci-e2e-test resourceVersion: "21135" uid: b6692f87-aae2-4fd9-8869-1cb3924f7f10 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:29:41.883172Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16657" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-8c20d-75b48d7b6d-dzgs8 to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:29:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:29:41Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c684501af03251 namespace: kserve-ci-e2e-test resourceVersion: "16664" uid: 25cbd209-3a75-4ffd-9df6-2ea1d0add70b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:29:42Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-8c20d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:42Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c6845027506129 namespace: kserve-ci-e2e-test resourceVersion: "16679" uid: c4938dd5-0e11-4612-ba3b-fab9d64121ed reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:42Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16668" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:29:42Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:29:42Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c68450586af935 namespace: kserve-ci-e2e-test resourceVersion: "16685" uid: c83b2abb-b873-41c6-9819-39abce0eda4a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8c20d} kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:29:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:42Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c6845059eb52c7 namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: f22c2c31-1bbe-41b2-a374-3f6b6c9bb61e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8c20d} kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:29:42Z" message: 'Created container: sequence-graph-8c20d' metadata: creationTimestamp: "2026-07-28T17:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:42Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c684505cbf511e namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 43b15373-a172-4e5b-86c0-e8eb36908d5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8c20d} kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:29:42Z" message: Started container sequence-graph-8c20d metadata: creationTimestamp: "2026-07-28T17:29:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:29:42Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c684505d2c2a52 namespace: kserve-ci-e2e-test resourceVersion: "16689" uid: 9eec3a2f-f5f9-44ee-b0c5-e27f762da44a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8c20d} kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Stopping container sequence-graph-8c20d metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:56Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c684c349151c9d namespace: kserve-ci-e2e-test resourceVersion: "20279" uid: 9c112972-3900-4266-a287-a166998e1896 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-8c20d} kind: Pod name: sequence-graph-8c20d-75b48d7b6d-dzgs8 namespace: kserve-ci-e2e-test resourceVersion: "16660" uid: b4be8c2c-41c8-48d9-9feb-0848236b55e5 kind: Event lastTimestamp: "2026-07-28T17:38:24Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:24Z" name: sequence-graph-8c20d-75b48d7b6d-dzgs8.18c684c3ecfd087c namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: 912468f2-f60e-4ef6-a0e3-fdcf163e0863 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-8c20d-75b48d7b6d namespace: kserve-ci-e2e-test resourceVersion: "16654" uid: 314f6ee9-3c01-49fc-8710-9a79cb4e9230 kind: Event lastTimestamp: "2026-07-28T17:29:41Z" message: 'Created pod: sequence-graph-8c20d-75b48d7b6d-dzgs8' metadata: creationTimestamp: "2026-07-28T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:41Z" name: sequence-graph-8c20d-75b48d7b6d.18c684501a8149c6 namespace: kserve-ci-e2e-test resourceVersion: "16659" uid: f4d68cc7-3e06-4e26-900d-cdf39cf49e58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 3da26549-271b-4b4b-ad50-11aa1b959e1f kind: Event lastTimestamp: "2026-07-28T17:29:41Z" message: Scaled up replica set sequence-graph-8c20d-75b48d7b6d from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:41Z" name: sequence-graph-8c20d.18c6845019bdee35 namespace: kserve-ci-e2e-test resourceVersion: "16655" uid: 4ba5e8f6-205e-4ef3-be87-829778a1a6a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16671" uid: 9cac9a5a-5906-42f4-9427-5acddf19c1d9 kind: Event lastTimestamp: "2026-07-28T17:29:41Z" message: InferenceGraph [sequence-graph-8c20d] is Ready metadata: creationTimestamp: "2026-07-28T17:29:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:29:41Z" name: sequence-graph-8c20d.18c684501f1b8c40 namespace: kserve-ci-e2e-test resourceVersion: "16673" uid: f1d328ff-fbb8-4116-8fa8-88a34915d6a9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: e39fa664-f388-4df0-9c35-84dc7662a851 kind: Event lastTimestamp: "2026-07-28T17:29:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:56Z" name: sequence-graph-8c20d.18c68453985804e0 namespace: kserve-ci-e2e-test resourceVersion: "16799" uid: 0edc70ca-c08e-4096-af23-2e92a5f5e4b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16653" uid: e39fa664-f388-4df0-9c35-84dc7662a851 kind: Event lastTimestamp: "2026-07-28T17:29:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:29:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:56Z" name: sequence-graph-8c20d.18c6845398b896af namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: bdcda5aa-5551-4292-bcf6-d5d0237790e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:30:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: e39fa664-f388-4df0-9c35-84dc7662a851 kind: Event lastTimestamp: "2026-07-28T17:30:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:30:11Z" name: sequence-graph-8c20d.18c68457177956d2 namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 1423e6ff-c8c2-4f6a-a290-d67a076b48df reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:30:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16800" uid: e39fa664-f388-4df0-9c35-84dc7662a851 kind: Event lastTimestamp: "2026-07-28T17:30:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:30:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:30:11Z" name: sequence-graph-8c20d.18c6845717dd2de1 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 42e912fd-5756-4f2e-930e-2768fac98c76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:40:07.151365Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21925" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c7add-656869bfd7-9qnqn to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:40:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:40:07Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1afd3a872 namespace: kserve-ci-e2e-test resourceVersion: "21930" uid: 1e9018e0-e7b7-4026-97e0-8480aedd9600 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:40:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-c7add-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:40:07Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1bd751d03 namespace: kserve-ci-e2e-test resourceVersion: "21946" uid: dc54e288-79c4-403b-a2ad-e011440f63ac reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:08Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21935" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:40:08Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:40:08Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1edce2661 namespace: kserve-ci-e2e-test resourceVersion: "21953" uid: ae2a9e45-896d-41d7-b733-e8354f3a3533 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7add} kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:40:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:40:08Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1ef0e490a namespace: kserve-ci-e2e-test resourceVersion: "21955" uid: e932ece4-a0bd-4787-8902-d4fa00b0d9b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7add} kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:40:08Z" message: 'Created container: sequence-graph-c7add' metadata: creationTimestamp: "2026-07-28T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:40:08Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1f1dd71de namespace: kserve-ci-e2e-test resourceVersion: "21956" uid: d8e641a6-3f09-420c-9f57-ab105945308c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7add} kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:40:08Z" message: Started container sequence-graph-c7add metadata: creationTimestamp: "2026-07-28T17:40:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:40:08Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c684e1f25343e5 namespace: kserve-ci-e2e-test resourceVersion: "21957" uid: caed87b0-8cd2-4aa7-844b-a7d33c4d9e3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7add} kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Stopping container sequence-graph-c7add metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:21Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c68554d3eaa21e namespace: kserve-ci-e2e-test resourceVersion: "25656" uid: c061f648-411f-4c78-a665-7ea9174b99d5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7add} kind: Pod name: sequence-graph-c7add-656869bfd7-9qnqn namespace: kserve-ci-e2e-test resourceVersion: "21928" uid: 04236009-f9d3-4a65-911a-b7029f085638 kind: Event lastTimestamp: "2026-07-28T17:48:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:50Z" name: sequence-graph-c7add-656869bfd7-9qnqn.18c685559a6b085c namespace: kserve-ci-e2e-test resourceVersion: "26354" uid: c12a8796-6622-4840-961d-62ef8e0a8192 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c7add-656869bfd7 namespace: kserve-ci-e2e-test resourceVersion: "21922" uid: 0da42b0b-c06c-4c51-a776-3605bbce8db0 kind: Event lastTimestamp: "2026-07-28T17:40:07Z" message: 'Created pod: sequence-graph-c7add-656869bfd7-9qnqn' metadata: creationTimestamp: "2026-07-28T17:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:07Z" name: sequence-graph-c7add-656869bfd7.18c684e1af34e38c namespace: kserve-ci-e2e-test resourceVersion: "21927" uid: c91fcb19-b7a0-469b-b526-7d283de41988 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c7add namespace: kserve-ci-e2e-test resourceVersion: "21920" uid: d1d895cb-170f-47fd-af60-113dde7a6e4e kind: Event lastTimestamp: "2026-07-28T17:40:07Z" message: Scaled up replica set sequence-graph-c7add-656869bfd7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:07Z" name: sequence-graph-c7add.18c684e1ae367a8d namespace: kserve-ci-e2e-test resourceVersion: "21923" uid: b4d8c202-7140-4f9c-8a1c-d0d34465a263 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:40:07Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c7add namespace: kserve-ci-e2e-test resourceVersion: "21939" uid: 45751c35-c361-40be-9d31-979d5921d246 kind: Event lastTimestamp: "2026-07-28T17:40:07Z" message: InferenceGraph [sequence-graph-c7add] is Ready metadata: creationTimestamp: "2026-07-28T17:40:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:40:07Z" name: sequence-graph-c7add.18c684e1b34e164f namespace: kserve-ci-e2e-test resourceVersion: "21941" uid: e3cf2da4-1974-4a94-b3a3-9ac7bcb70305 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:40:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c7add namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: bf59aa84-06bc-48ae-b21c-4375c5007de0 kind: Event lastTimestamp: "2026-07-28T17:40:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:37Z" name: sequence-graph-c7add.18c684e52cb66e8d namespace: kserve-ci-e2e-test resourceVersion: "22169" uid: f6cd1d6f-1147-482d-92a6-2b6cb2d4b27f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:40:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c7add namespace: kserve-ci-e2e-test resourceVersion: "21921" uid: bf59aa84-06bc-48ae-b21c-4375c5007de0 kind: Event lastTimestamp: "2026-07-28T17:40:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:37Z" name: sequence-graph-c7add.18c684e52d22ab23 namespace: kserve-ci-e2e-test resourceVersion: "22170" uid: 3f19e746-5d50-44d8-836a-380b9328b8cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:48:36.069109Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26068" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-32bdd-ff74654bc-bvqc4 to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c685582db01fbd namespace: kserve-ci-e2e-test resourceVersion: "26074" uid: a171691d-810f-4a7d-9ec6-a757edff604d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26077" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c68558476a19e1 namespace: kserve-ci-e2e-test resourceVersion: "26090" uid: 4cbb49c8-5a11-4f40-a63a-3fde3866a765 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32bdd} kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c6855848c30bee namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 93a1c844-df23-4b2b-a6dd-af7aa043c351 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32bdd} kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'Created container: splitter-graph-32bdd' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c685584b77fb05 namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: c058d7f2-aa62-4442-b6f6-cb97786007fc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32bdd} kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: Started container splitter-graph-32bdd metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c685584be52aa3 namespace: kserve-ci-e2e-test resourceVersion: "26095" uid: 8a65763c-cca9-4365-a0c0-0cd9cb8c0d85 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32bdd} kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Stopping container splitter-graph-32bdd metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c6855a86e1813a namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: 42b33170-3e91-48db-86a2-e9e62a1ab118 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:48:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-32bdd} kind: Pod name: splitter-graph-32bdd-ff74654bc-bvqc4 namespace: kserve-ci-e2e-test resourceVersion: "26071" uid: 1b5c3262-e419-4778-9fa9-41c71ccc983f kind: Event lastTimestamp: "2026-07-28T17:49:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:48:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:13Z" name: splitter-graph-32bdd-ff74654bc-bvqc4.18c6855afe7e7b78 namespace: kserve-ci-e2e-test resourceVersion: "26547" uid: 78bb8bd4-32a9-4208-bf1d-4162f4e651ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-32bdd-ff74654bc namespace: kserve-ci-e2e-test resourceVersion: "26065" uid: 9b4dc488-5f0d-41de-9c72-c3e62766b3c1 kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'Created pod: splitter-graph-32bdd-ff74654bc-bvqc4' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd-ff74654bc.18c685582d35f8bb namespace: kserve-ci-e2e-test resourceVersion: "26069" uid: 88d1ed4d-717a-45cd-8930-b94e3684ec23 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-32bdd namespace: kserve-ci-e2e-test resourceVersion: "26063" uid: 7f73cb0c-fe31-46d2-8df0-01a22d24f13f kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: Scaled up replica set splitter-graph-32bdd-ff74654bc from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd.18c685582ca1e41c namespace: kserve-ci-e2e-test resourceVersion: "26067" uid: dd37ef6a-c90d-4b6f-92c2-149e886b8d1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-32bdd namespace: kserve-ci-e2e-test resourceVersion: "26082" uid: 9d92704a-646f-4b72-accd-96706e3fa47d kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: InferenceGraph [splitter-graph-32bdd] is Ready metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:36Z" name: splitter-graph-32bdd.18c68558320d5371 namespace: kserve-ci-e2e-test resourceVersion: "26085" uid: 57886686-652e-4ead-973f-9a76b47644a3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:49:56.422491Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26966" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-80e89-5f679bbbf7-v744v to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c6856ae31f7687 namespace: kserve-ci-e2e-test resourceVersion: "26971" uid: b3236bb4-5e26-4cfa-a0f9-388a317cb33e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26974" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: Add eth0 [10.133.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c6856afcbe598d namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 0d4ccf0d-43c1-4f07-bd0c-4b181855836d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-80e89} kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c6856afe141bc0 namespace: kserve-ci-e2e-test resourceVersion: "26986" uid: abd4f19a-763e-49a6-aca0-d1141efcfcea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-80e89} kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: 'Created container: splitter-graph-80e89' metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c6856b00e79b26 namespace: kserve-ci-e2e-test resourceVersion: "26988" uid: c2c7dc1c-fdba-43d8-8abf-542ed9fe0491 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-80e89} kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: Started container splitter-graph-80e89 metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c6856b0155d82e namespace: kserve-ci-e2e-test resourceVersion: "26989" uid: a8b4dd6f-5629-42ba-97d7-c16c3877a327 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-80e89} kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:58:11Z" message: Stopping container splitter-graph-80e89 metadata: creationTimestamp: "2026-07-28T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:11Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c685de0fd5fe61 namespace: kserve-ci-e2e-test resourceVersion: "30430" uid: 88b4d1f0-70d3-4f61-9262-fc8dbf946605 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-80e89} kind: Pod name: splitter-graph-80e89-5f679bbbf7-v744v namespace: kserve-ci-e2e-test resourceVersion: "26968" uid: ab0078cf-f72f-43a9-a7e2-7e856296655e kind: Event lastTimestamp: "2026-07-28T17:58:38Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:38Z" name: splitter-graph-80e89-5f679bbbf7-v744v.18c685dea01e764c namespace: kserve-ci-e2e-test resourceVersion: "30664" uid: 7b4f6638-7e3c-4b86-a93b-2733ab3a035a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-80e89-5f679bbbf7 namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: a5e8cec4-fd9d-4925-a3d5-7508d15ab2a4 kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: 'Created pod: splitter-graph-80e89-5f679bbbf7-v744v' metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89-5f679bbbf7.18c6856ae292c16d namespace: kserve-ci-e2e-test resourceVersion: "26967" uid: 43a87756-ae5d-408e-a0a3-1537d276553b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: 61395f37-2e8f-491f-acf4-ec70f058190f kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: Scaled up replica set splitter-graph-80e89-5f679bbbf7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89.18c6856ae1da080c namespace: kserve-ci-e2e-test resourceVersion: "26964" uid: 146aaf12-d4da-4586-803c-65b8aaf3aaa8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26978" uid: 8877dbf9-a5d4-472a-ba2d-f54d7c58d422 kind: Event lastTimestamp: "2026-07-28T17:49:56Z" message: InferenceGraph [splitter-graph-80e89] is Ready metadata: creationTimestamp: "2026-07-28T17:49:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:56Z" name: splitter-graph-80e89.18c6856ae7accf02 namespace: kserve-ci-e2e-test resourceVersion: "26981" uid: f9c0a947-f954-4ba0-8ecf-1c4cd7d418e9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: c237a644-bb0b-4eac-b4d6-4d5ddde697ae kind: Event lastTimestamp: "2026-07-28T17:50:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:11Z" name: splitter-graph-80e89.18c6856e6064b944 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: 0ab21e07-0a0a-4191-af61-ecf7cc403972 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26962" uid: c237a644-bb0b-4eac-b4d6-4d5ddde697ae kind: Event lastTimestamp: "2026-07-28T17:50:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:50:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:11Z" name: splitter-graph-80e89.18c6856e60cd09a8 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 3cc23678-35a9-4028-b38f-3b45d708632f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: c237a644-bb0b-4eac-b4d6-4d5ddde697ae kind: Event lastTimestamp: "2026-07-28T17:50:26Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:26Z" name: splitter-graph-80e89.18c68571df9deffc namespace: kserve-ci-e2e-test resourceVersion: "27212" uid: b3abcf39-4851-4776-aa8a-695b3734bc31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-80e89 namespace: kserve-ci-e2e-test resourceVersion: "27106" uid: c237a644-bb0b-4eac-b4d6-4d5ddde697ae kind: Event lastTimestamp: "2026-07-28T17:50:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:50:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:26Z" name: splitter-graph-80e89.18c68571e012db8c namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: 9db61384-2bf1-4b50-9ecb-943a12cf2316 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:47:36.076362Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25071" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-32bdd-predictor-68f496d98f-tfnht to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a35d791c3 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: afc491ee-9235-44b1-b8b4-bab4ef426809 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a4fea7362 namespace: kserve-ci-e2e-test resourceVersion: "25119" uid: 1b20eb20-b340-4c41-aa04-dd7266e4b75c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a5142e155 namespace: kserve-ci-e2e-test resourceVersion: "25121" uid: ee134f25-dff0-4788-987e-c69817df23e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a541f80ba namespace: kserve-ci-e2e-test resourceVersion: "25122" uid: 815e626d-9863-42f9-b8b4-32e705ceafb5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a5493ab8a namespace: kserve-ci-e2e-test resourceVersion: "25123" uid: f89e10f3-a20b-45e4-8705-39f0f32ae3cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a549e22a1 namespace: kserve-ci-e2e-test resourceVersion: "25124" uid: 0fec7af6-5d45-4ba1-b615-1884256e363b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a57815e08 namespace: kserve-ci-e2e-test resourceVersion: "25125" uid: 00412b44-7505-44f4-95c5-fc397714cbcb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854a58060a19 namespace: kserve-ci-e2e-test resourceVersion: "25126" uid: 5c176057-5d30-4d65-95ae-644e15f1c2cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:48:14Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:14Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6854ac46c67fd namespace: kserve-ci-e2e-test resourceVersion: "25436" uid: 5b336f80-84f0-4609-98c6-74720a7097e3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6855a8f3aade1 namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: 670cd81f-7bcc-4e9f-b3b9-469d55bbc2ce reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6855a8f3b600d namespace: kserve-ci-e2e-test resourceVersion: "26236" uid: c3f93f64-e5a2-4e64-a25d-8d22a4096911 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht namespace: kserve-ci-e2e-test resourceVersion: "25074" uid: 13baaa32-4f7e-42e5-8d7f-e5020b670207 kind: Event lastTimestamp: "2026-07-28T17:48:49Z" message: 'Readiness probe failed: Get "https://10.132.0.36:8643/healthz": dial tcp 10.132.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:49Z" name: success-200-isvc-32bdd-predictor-68f496d98f-tfnht.18c6855b4cae398b namespace: kserve-ci-e2e-test resourceVersion: "26340" uid: 74ab34db-b82b-47ee-9f16-89d007dbcc06 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-32bdd-predictor-68f496d98f namespace: kserve-ci-e2e-test resourceVersion: "25051" uid: 0b65ae76-1c5c-495d-8442-838eebf31cd0 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: 'Created pod: success-200-isvc-32bdd-predictor-68f496d98f-tfnht' metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor-68f496d98f.18c6854a3560ae62 namespace: kserve-ci-e2e-test resourceVersion: "25073" uid: f133b865-675b-4997-af20-8d42392f12e6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25049" uid: c53381ab-6b2a-4274-bab5-35e40861d529 kind: Event lastTimestamp: "2026-07-28T17:47:36Z" message: Scaled up replica set success-200-isvc-32bdd-predictor-68f496d98f from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:47:36Z" name: success-200-isvc-32bdd-predictor.18c6854a317f5999 namespace: kserve-ci-e2e-test resourceVersion: "25052" uid: 9200a8a7-c926-4f4c-b0ec-231ae1d5482b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:47:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 817a70f4-cbf7-4482-b114-00d5575fcd9a kind: Event lastTimestamp: "2026-07-28T17:48:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:06Z" name: success-200-isvc-32bdd-predictor.18c6854db022eae6 namespace: kserve-ci-e2e-test resourceVersion: "25377" uid: 214beba6-b027-4e0c-a165-d6708638671c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:47:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25050" uid: 817a70f4-cbf7-4482-b114-00d5575fcd9a kind: Event lastTimestamp: "2026-07-28T17:48:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:47:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:06Z" name: success-200-isvc-32bdd-predictor.18c6854db10d2fa2 namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 415f0dbb-f96a-4661-a086-8c129ac30a90 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25257" uid: 817a70f4-cbf7-4482-b114-00d5575fcd9a kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: success-200-isvc-32bdd-predictor.18c68554aef237bc namespace: kserve-ci-e2e-test resourceVersion: "26078" uid: 8b8df9d7-9319-47eb-842c-31f21af63ed4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-32bdd-predictor namespace: kserve-ci-e2e-test resourceVersion: "25257" uid: 817a70f4-cbf7-4482-b114-00d5575fcd9a kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: success-200-isvc-32bdd-predictor.18c68554af71413a namespace: kserve-ci-e2e-test resourceVersion: "26079" uid: f6c1ea82-e45e-4885-9440-0b2044039f28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:47:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32bdd namespace: kserve-ci-e2e-test resourceVersion: "25037" uid: 09301029-8c2c-42cd-aead-ca9e9a61289e kind: Event lastTimestamp: "2026-07-28T17:48:24Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:47:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:24Z" name: success-200-isvc-32bdd.18c6854a319087a6 namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: 80203a79-ec1c-4f94-8a5b-d7160141d2c3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:24Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-32bdd namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: 09301029-8c2c-42cd-aead-ca9e9a61289e kind: Event lastTimestamp: "2026-07-28T17:48:24Z" message: InferenceService [success-200-isvc-32bdd] is Ready metadata: creationTimestamp: "2026-07-28T17:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:24Z" name: success-200-isvc-32bdd.18c6855580934144 namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: ef489880-1395-4886-baa0-372b342840b5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:37:56.757001Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20332" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4108d-predictor-6fdcc84849-ntwgz to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c353b78d8d namespace: kserve-ci-e2e-test resourceVersion: "20338" uid: 3512dd2b-50cf-4525-8518-5e0fb7c216d2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c36e9edc11 namespace: kserve-ci-e2e-test resourceVersion: "20385" uid: d9902216-1f23-494d-8a77-01cbd947c15f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c36ff26e6e namespace: kserve-ci-e2e-test resourceVersion: "20388" uid: 724616f4-21d8-4375-9444-ee63f5967f40 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c372f8d049 namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 46d7e247-7774-47b2-a8fa-6a09e039b86f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c3736b7102 namespace: kserve-ci-e2e-test resourceVersion: "20390" uid: 5eb5e01c-a68a-4774-a5c9-24cc02870594 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c373765c25 namespace: kserve-ci-e2e-test resourceVersion: "20391" uid: 868842a0-8178-4f0e-97e0-497495c09fd1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c376342a5c namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: e38b369f-bc8f-4c41-a291-24057ec592c6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:37:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:57Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c376bd9505 namespace: kserve-ci-e2e-test resourceVersion: "20393" uid: f3919994-c1a9-40a0-8d7e-f77fa01d3ae7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:38:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:35Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684c3e46ea5b8 namespace: kserve-ci-e2e-test resourceVersion: "20921" uid: c2c4ff93-e3d5-4ac4-b053-de2a90d05c01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684d3ae9fe319 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 4dfa88ba-940a-4c7b-ad61-16153c2b6baf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684d3aea08cc7 namespace: kserve-ci-e2e-test resourceVersion: "21267" uid: 248d2b14-a03d-4c35-85cf-ee4c695c7a97 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz namespace: kserve-ci-e2e-test resourceVersion: "20335" uid: b0c82467-5c68-4d9e-bf2f-599d5c8e37d3 kind: Event lastTimestamp: "2026-07-28T17:39:10Z" message: 'Readiness probe failed: Get "https://10.132.0.30:8643/healthz": dial tcp 10.132.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:39:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:10Z" name: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz.18c684d46d18c85b namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: fd294018-0e49-4804-bbd2-51e45bc67379 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4108d-predictor-6fdcc84849 namespace: kserve-ci-e2e-test resourceVersion: "20323" uid: d8890ede-9434-448d-bbe7-14dc818a6248 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'Created pod: success-200-isvc-4108d-predictor-6fdcc84849-ntwgz' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-4108d-predictor-6fdcc84849.18c684c35357c90c namespace: kserve-ci-e2e-test resourceVersion: "20334" uid: 80c86681-5ec0-4d1f-9e6b-6bb2d6e89762 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20319" uid: 463fbf3d-f6fe-4595-a558-e81c7e7f6c5d kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Scaled up replica set success-200-isvc-4108d-predictor-6fdcc84849 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-4108d-predictor.18c684c351949aca namespace: kserve-ci-e2e-test resourceVersion: "20325" uid: 84ecf8ce-5331-4cdf-8062-20330ebde394 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 9efb6957-bf8c-481b-9910-2cabf496e736 kind: Event lastTimestamp: "2026-07-28T17:38:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:41Z" name: success-200-isvc-4108d-predictor.18c684c6d05bfcaa namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 74b3a8a5-6641-4b5c-8a58-84bc2484f66f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20322" uid: 9efb6957-bf8c-481b-9910-2cabf496e736 kind: Event lastTimestamp: "2026-07-28T17:38:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:41Z" name: success-200-isvc-4108d-predictor.18c684c6d0e002bd namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: 744682fe-aa7f-49c1-aee3-b28181aa1500 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9efb6957-bf8c-481b-9910-2cabf496e736 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: success-200-isvc-4108d-predictor.18c684d14d1ebb29 namespace: kserve-ci-e2e-test resourceVersion: "21121" uid: 0e64afd9-22ea-40d7-9ced-bdb40528801d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4108d-predictor namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 9efb6957-bf8c-481b-9910-2cabf496e736 kind: Event lastTimestamp: "2026-07-28T17:38:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:56Z" name: success-200-isvc-4108d-predictor.18c684d14dab35a6 namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: f5ea3fa6-da73-49dc-9683-74e397aae05d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: d2bd4827-d793-494f-9ce6-e56de85bee22 kind: Event lastTimestamp: "2026-07-28T17:38:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:45Z" name: success-200-isvc-4108d.18c684c351dfc4b8 namespace: kserve-ci-e2e-test resourceVersion: "21009" uid: 5125133b-ed0f-454e-b24a-dda1ed2c7d94 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: d2bd4827-d793-494f-9ce6-e56de85bee22 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-4108d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4108d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-4108d.18c684c352395db0 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: b5609daf-fdb2-4a1c-a2f0-8ef4c23de09e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "20297" uid: d2bd4827-d793-494f-9ce6-e56de85bee22 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4108d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-4108d.18c684c35239ba9f namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: f50b75a8-472f-4cae-b332-9368776eed1b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4108d namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: d2bd4827-d793-494f-9ce6-e56de85bee22 kind: Event lastTimestamp: "2026-07-28T17:38:45Z" message: InferenceService [success-200-isvc-4108d] is Ready metadata: creationTimestamp: "2026-07-28T17:38:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:45Z" name: success-200-isvc-4108d.18c684cea0a55821 namespace: kserve-ci-e2e-test resourceVersion: "21008" uid: 1dce262c-5350-49e3-9cd3-ead4d230b282 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:37:00.775729Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:37:00Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b64af92009 namespace: kserve-ci-e2e-test resourceVersion: "19754" uid: 26891c13-3c4e-46bc-8c0d-feba4e176c45 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b664a2cff3 namespace: kserve-ci-e2e-test resourceVersion: "19799" uid: 5765f79a-21be-4e9e-b1df-252bdb5fc312 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b665f92057 namespace: kserve-ci-e2e-test resourceVersion: "19801" uid: 38e93b00-88d9-4591-bca7-f7d60bf5ea95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b668f94a37 namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: 3c2ea7fa-ef8a-453a-b5df-c5818d690610 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b6696d189a namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 927cf3f5-4124-44c3-a82a-40a83dc8adf5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b66977640e namespace: kserve-ci-e2e-test resourceVersion: "19805" uid: 88b907c6-3a1c-499d-95fb-0f16430cb305 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b66c628046 namespace: kserve-ci-e2e-test resourceVersion: "19806" uid: 5c5fcdc3-81a9-47b6-8d8e-92968f3ce4fe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:01Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:01Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b66cd8cfb3 namespace: kserve-ci-e2e-test resourceVersion: "19807" uid: b75fe26b-ab0e-46ba-9e51-3a093b1eb8a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:37:37Z" message: 'Readiness probe failed: dial tcp 10.132.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:37Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684b6901e86ba namespace: kserve-ci-e2e-test resourceVersion: "20119" uid: 985217d4-0568-4e37-a062-76cb6bfa93b5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684c6a674bd4b namespace: kserve-ci-e2e-test resourceVersion: "20628" uid: 0154c730-77d7-41a3-992f-6ada2401a1bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684c6a6755919 namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 4d0a631e-4da4-43b2-9849-764a4f915d14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb namespace: kserve-ci-e2e-test resourceVersion: "19750" uid: 4ad04571-d18c-4f82-a5cf-619bf490310e kind: Event lastTimestamp: "2026-07-28T17:38:12Z" message: 'Readiness probe failed: Get "https://10.132.0.28:8643/healthz": dial tcp 10.132.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:12Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb.18c684c71873be21 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: d2ab6ce0-aaea-499b-9aa9-57471c0037fa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-59e5e-predictor-5984bd5fb5 namespace: kserve-ci-e2e-test resourceVersion: "19725" uid: c91728a6-1f70-4c04-b750-aadf2a3988fc kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: 'Created pod: success-200-isvc-59e5e-predictor-5984bd5fb5-xqkfb' metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:00Z" name: success-200-isvc-59e5e-predictor-5984bd5fb5.18c684b64a999485 namespace: kserve-ci-e2e-test resourceVersion: "19751" uid: 3ae63e2b-5006-4d19-9a29-2c8110eb439e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 41f18f87-c94f-49f6-b4d9-f416eb1aa877 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Scaled up replica set success-200-isvc-59e5e-predictor-5984bd5fb5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:00Z" name: success-200-isvc-59e5e-predictor.18c684b646ff0af1 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 2d675180-5d1c-4861-aa7d-36a5a6ca0074 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 0aead0ce-81c5-426e-973f-7ba48f61f9e3 kind: Event lastTimestamp: "2026-07-28T17:37:30Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:30Z" name: success-200-isvc-59e5e-predictor.18c684b9c567f371 namespace: kserve-ci-e2e-test resourceVersion: "20060" uid: e1b1074a-6cc8-4899-8e69-3ebefafe4163 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 0aead0ce-81c5-426e-973f-7ba48f61f9e3 kind: Event lastTimestamp: "2026-07-28T17:37:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:37:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:37:30Z" name: success-200-isvc-59e5e-predictor.18c684b9c5cd07c6 namespace: kserve-ci-e2e-test resourceVersion: "20061" uid: 8f2d9ca5-ff4a-4512-9c34-3b2ed6497a5d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 0aead0ce-81c5-426e-973f-7ba48f61f9e3 kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: success-200-isvc-59e5e-predictor.18c684c0c38077b4 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: baa50436-7071-4424-bdca-f6e5955d6726 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:37:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-59e5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19946" uid: 0aead0ce-81c5-426e-973f-7ba48f61f9e3 kind: Event lastTimestamp: "2026-07-28T17:38:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:37:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:00Z" name: success-200-isvc-59e5e-predictor.18c684c0c3e588b7 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 38e37e2e-fc40-412c-a3df-d039d3b3bda7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "19712" uid: 1ee7d6a8-8f13-4344-8923-731aa8bb3987 kind: Event lastTimestamp: "2026-07-28T17:37:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:48Z" name: success-200-isvc-59e5e.18c684b64708f881 namespace: kserve-ci-e2e-test resourceVersion: "20201" uid: b130dae9-23f6-4cc8-863e-9cb83fdb1b0d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-59e5e namespace: kserve-ci-e2e-test resourceVersion: "20199" uid: 1ee7d6a8-8f13-4344-8923-731aa8bb3987 kind: Event lastTimestamp: "2026-07-28T17:37:48Z" message: InferenceService [success-200-isvc-59e5e] is Ready metadata: creationTimestamp: "2026-07-28T17:37:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:37:48Z" name: success-200-isvc-59e5e.18c684c14b7dd2e5 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: 0b7295f7-05cc-470e-ae7b-8483329fb99a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:27:45.916637Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15353" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-65026-predictor-7ff85d8cb5-7742f to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:27:45Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684351acb5911 namespace: kserve-ci-e2e-test resourceVersion: "15362" uid: 63185e78-dcc5-40a1-bc6b-16b16398bc3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-65026-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:46Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c68435268be1a1 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: 147e18f0-e538-408a-9147-79eff81655e1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15368" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:46Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:27:46Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684355876934b namespace: kserve-ci-e2e-test resourceVersion: "15413" uid: e0f526d0-3ed4-4b49-8244-cf0d71dba3ed reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:46Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684355a51ee4d namespace: kserve-ci-e2e-test resourceVersion: "15416" uid: a4395735-01e6-4d49-865d-48602ec49d6c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684355d27eb1f namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 4d273b0e-ae2e-4b49-b0b5-2cadd4e91458 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684355dba8a9e namespace: kserve-ci-e2e-test resourceVersion: "15418" uid: b256836f-fba9-4ceb-b750-e372f899b790 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684355dc5f24e namespace: kserve-ci-e2e-test resourceVersion: "15419" uid: 43901a98-d7b4-482a-bb7b-85f1d64e2c4a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c68435608480ce namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 45a5574c-477a-41e7-919d-c0587d73f9d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c6843560e94903 namespace: kserve-ci-e2e-test resourceVersion: "15424" uid: 486b8ad5-87f4-4bd9-ae1f-76e8e036f676 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:37:03Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:03Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684358bc04e90 namespace: kserve-ci-e2e-test resourceVersion: "19846" uid: 0b29145a-96ea-41a8-be5a-06bf25ab77f4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:00Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684b643ae4a5c namespace: kserve-ci-e2e-test resourceVersion: "19713" uid: 23011638-68b2-4f94-bea6-39431b999a44 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:00Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684b643aefea8 namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 12c22f8b-1bef-44fe-9ed4-63ee02f6221b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f namespace: kserve-ci-e2e-test resourceVersion: "15357" uid: 0e987338-9f49-4c49-9b84-cf291a6bd8e4 kind: Event lastTimestamp: "2026-07-28T17:37:03Z" message: 'Readiness probe failed: Get "https://10.132.0.24:8643/healthz": dial tcp 10.132.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:03Z" name: success-200-isvc-65026-predictor-7ff85d8cb5-7742f.18c684b70038dddc namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: a9c3f0bf-34bb-4a63-b39e-0b4c404e79e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-65026-predictor-7ff85d8cb5 namespace: kserve-ci-e2e-test resourceVersion: "15331" uid: 4b23f81c-e5b7-4051-b217-5b1c551e102b kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: 'Created pod: success-200-isvc-65026-predictor-7ff85d8cb5-7742f' metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:45Z" name: success-200-isvc-65026-predictor-7ff85d8cb5.18c684351a52ece1 namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: afd6bccf-3bd1-478c-ad33-def8a7c0c8a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: 416aa59c-ce7a-49ba-a432-fd95e8c93ec6 kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Scaled up replica set success-200-isvc-65026-predictor-7ff85d8cb5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:45Z" name: success-200-isvc-65026-predictor.18c684351740395f namespace: kserve-ci-e2e-test resourceVersion: "15334" uid: ca2021a5-f279-4634-b297-1cfae76c02a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: b04223fa-05f7-411d-b713-5f1e7533e6d4 kind: Event lastTimestamp: "2026-07-28T17:28:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:15Z" name: success-200-isvc-65026-predictor.18c68438963a11bf namespace: kserve-ci-e2e-test resourceVersion: "15704" uid: ecbc2299-1aa2-48b2-ab38-029f561e6d6c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: b04223fa-05f7-411d-b713-5f1e7533e6d4 kind: Event lastTimestamp: "2026-07-28T17:28:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:15Z" name: success-200-isvc-65026-predictor.18c6843896a6ba4d namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: cace0f67-09f8-4e29-8d3b-d1a1e6b0c747 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: b04223fa-05f7-411d-b713-5f1e7533e6d4 kind: Event lastTimestamp: "2026-07-28T17:29:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:00Z" name: success-200-isvc-65026-predictor.18c6843f948cd60c namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: b4d1d916-dccf-4df4-852e-dd5de20d8560 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:28:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-65026-predictor namespace: kserve-ci-e2e-test resourceVersion: "15565" uid: b04223fa-05f7-411d-b713-5f1e7533e6d4 kind: Event lastTimestamp: "2026-07-28T17:29:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:28:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:00Z" name: success-200-isvc-65026-predictor.18c6843f94fb3b61 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: b6c1c404-102b-4e8c-b412-ec527993173f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15311" uid: 7b4a8a8c-27e1-4ce7-aeb9-b45eb37e95cd kind: Event lastTimestamp: "2026-07-28T17:28:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:34Z" name: success-200-isvc-65026.18c684351772a46b namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: ff6bbff4-fdd8-443b-8ed8-0cbcc32fcc4a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-65026 namespace: kserve-ci-e2e-test resourceVersion: "15900" uid: 7b4a8a8c-27e1-4ce7-aeb9-b45eb37e95cd kind: Event lastTimestamp: "2026-07-28T17:28:33Z" message: InferenceService [success-200-isvc-65026] is Ready metadata: creationTimestamp: "2026-07-28T17:28:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:33Z" name: success-200-isvc-65026.18c684404b1b7886 namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: 528cc312-1954-4289-a2a9-6b976de4999b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:48:46.351522Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-80e89-predictor-7b866dd6f9-smth2 to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855a929160b3 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: 6ce4b3d1-a087-4894-9fc0-db7de6b71c67 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26255" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855aace807f1 namespace: kserve-ci-e2e-test resourceVersion: "26292" uid: 2ccda92b-d6ff-4f8e-8c93-e1d950941d2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855aaea89cf0 namespace: kserve-ci-e2e-test resourceVersion: "26295" uid: 05bb3c27-4207-497e-b417-37fe450af072 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ab16b10de namespace: kserve-ci-e2e-test resourceVersion: "26297" uid: 888575ed-3535-4c89-b9c4-a8b9a1b9e03d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ab1de881d namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: d4cf84c6-fc1a-48af-83a4-d3d867e265a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ab1e7c9bb namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: a4aace78-10b8-4a5f-be5a-7f4b69ba59ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ab4b0f6ba namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 425ef7d0-d514-4b02-982d-08604bd19e7a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ab526df3c namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 5403ecef-d4e0-465d-8241-0a89268308b8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T17:48:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:58:13Z" message: 'Readiness probe failed: dial tcp 10.132.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:13Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c6855ae6e2e11e namespace: kserve-ci-e2e-test resourceVersion: "30482" uid: cd4d5746-9522-4bc4-a6dc-90b8fc8e087e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:58:11Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:11Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c685de18461d3d namespace: kserve-ci-e2e-test resourceVersion: "30447" uid: aaa66c71-4154-456b-9fe6-7b38b2769535 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:58:11Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:58:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:11Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c685de1846958d namespace: kserve-ci-e2e-test resourceVersion: "30449" uid: 1a85f99c-26cd-49df-ad08-d469a21457f0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:58:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2 namespace: kserve-ci-e2e-test resourceVersion: "26248" uid: 2c8be5a8-0cef-4fb8-9289-c32719c96da1 kind: Event lastTimestamp: "2026-07-28T17:58:13Z" message: 'Readiness probe failed: Get "https://10.132.0.40:8643/healthz": dial tcp 10.132.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:58:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:58:13Z" name: success-200-isvc-80e89-predictor-7b866dd6f9-smth2.18c685deaf6a3ac3 namespace: kserve-ci-e2e-test resourceVersion: "30480" uid: d083e24a-4333-4759-ad13-85c8579cb48f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-80e89-predictor-7b866dd6f9 namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: 74a81e93-88d7-4bc1-ba75-5735981ece9c kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Created pod: success-200-isvc-80e89-predictor-7b866dd6f9-smth2' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor-7b866dd6f9.18c6855a91e996fb namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: 380ac6c7-2b7a-4968-9c60-d45e76b32d09 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: c3f1210f-2328-44eb-b18f-5aaf6bc8668e kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: Scaled up replica set success-200-isvc-80e89-predictor-7b866dd6f9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89-predictor.18c6855a8fdbd993 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: d1e8c955-ce68-436a-9c18-f08a9c35e65e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 7990dfc2-4cd0-46e7-9e16-f86880e0e299 kind: Event lastTimestamp: "2026-07-28T17:50:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:01Z" name: success-200-isvc-80e89-predictor.18c6855e0ec8bef3 namespace: kserve-ci-e2e-test resourceVersion: "27020" uid: e88cd55d-4a87-4982-bb2e-b1b3facd33ec reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:49:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-80e89-predictor namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 7990dfc2-4cd0-46e7-9e16-f86880e0e299 kind: Event lastTimestamp: "2026-07-28T17:50:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:49:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:01Z" name: success-200-isvc-80e89-predictor.18c6855e0f40289b namespace: kserve-ci-e2e-test resourceVersion: "27021" uid: f221f334-3286-4a39-bd0c-75c284952ab5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 2d79a9ed-eda0-41e8-afd1-e54833220374 kind: Event lastTimestamp: "2026-07-28T17:49:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:44Z" name: success-200-isvc-80e89.18c6855a8fef5ef2 namespace: kserve-ci-e2e-test resourceVersion: "26861" uid: f2769eb1-5c9f-40cc-8e94-b61959031ae0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 2d79a9ed-eda0-41e8-afd1-e54833220374 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'Failed to update status for InferenceService "success-200-isvc-80e89": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-80e89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89.18c6855a9085921b namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: 6f6ec618-d5ab-4cd5-8362-2969edd56cda reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26208" uid: 2d79a9ed-eda0-41e8-afd1-e54833220374 kind: Event lastTimestamp: "2026-07-28T17:48:46Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-80e89": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:46Z" name: success-200-isvc-80e89.18c6855a9085ee78 namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 5710903a-7541-43f6-8549-81a986199719 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-80e89 namespace: kserve-ci-e2e-test resourceVersion: "26855" uid: 2d79a9ed-eda0-41e8-afd1-e54833220374 kind: Event lastTimestamp: "2026-07-28T17:49:43Z" message: InferenceService [success-200-isvc-80e89] is Ready metadata: creationTimestamp: "2026-07-28T17:49:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:43Z" name: success-200-isvc-80e89.18c68567fbb2b0ed namespace: kserve-ci-e2e-test resourceVersion: "26857" uid: ade4bc2d-1bba-4998-8d8f-65ca6acd5721 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:28:41.861991Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16028" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:28:41Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684422165af55 namespace: kserve-ci-e2e-test resourceVersion: "16038" uid: 9945e436-494c-4a4d-b289-843b76fc7fbe reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-8c20d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:41Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c6844227c6b0d6 namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: e60abffe-ef7d-45cb-89cf-587d4fb68354 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16048" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684424c9f8e7b namespace: kserve-ci-e2e-test resourceVersion: "16100" uid: 3d841f4b-0626-497c-a719-81b5f98a8c5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684424e0d8b9f namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 2b47b810-6846-47f2-bfca-7d6a21368d8e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c6844251043045 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: e06f1a01-541a-4619-a532-61f46892c60a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c6844251692384 namespace: kserve-ci-e2e-test resourceVersion: "16104" uid: e3ab5701-4acc-44f2-9851-7768b7ffe1c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c6844251737899 namespace: kserve-ci-e2e-test resourceVersion: "16105" uid: cf751d9d-d33e-45c8-880c-0744dce42efe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684425416ee10 namespace: kserve-ci-e2e-test resourceVersion: "16106" uid: 51a1ccea-dd79-4f49-b27a-afc02a3ab138 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:28:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:42Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c68442548094a7 namespace: kserve-ci-e2e-test resourceVersion: "16107" uid: 6f6a582a-1684-4d30-addb-5577c35757e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:28:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:37:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:28:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:59Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c68442672f8aa9 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 704315ac-7059-4051-b1b8-53993c81fd3d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684c350fd4b21 namespace: kserve-ci-e2e-test resourceVersion: "20321" uid: 90dd11e2-7d7b-4c1b-96c7-9268765b2e39 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:37:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:56Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684c350fddfb1 namespace: kserve-ci-e2e-test resourceVersion: "20324" uid: 7899bee7-5149-4664-b992-a5916e824fba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq namespace: kserve-ci-e2e-test resourceVersion: "16033" uid: 12350e64-4ab0-4772-a117-fdccece9f484 kind: Event lastTimestamp: "2026-07-28T17:37:59Z" message: 'Readiness probe failed: Get "https://10.132.0.26:8643/healthz": dial tcp 10.132.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:37:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:59Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq.18c684c3dbc23af1 namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: dee8132f-0aa0-4213-a414-f13ae0a14490 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-8c20d-predictor-f89cb6ff7 namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: b965e882-1d79-40bc-a3f7-d227a0a6fd94 kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: 'Created pod: success-200-isvc-8c20d-predictor-f89cb6ff7-8cwdq' metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: success-200-isvc-8c20d-predictor-f89cb6ff7.18c6844220d660d7 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: af63b341-6ce7-425e-b371-abc96cff4b59 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16005" uid: da7e2913-ee71-4e1a-b2c4-ec0f7743caba kind: Event lastTimestamp: "2026-07-28T17:28:41Z" message: Scaled up replica set success-200-isvc-8c20d-predictor-f89cb6ff7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:41Z" name: success-200-isvc-8c20d-predictor.18c684421eb59890 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 6e9ad521-2f16-407d-b0e8-347e63b29d28 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: bb7c2cb8-3e79-4e8e-9872-2d886e9c6eae kind: Event lastTimestamp: "2026-07-28T17:29:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:11Z" name: success-200-isvc-8c20d-predictor.18c684459d3ab709 namespace: kserve-ci-e2e-test resourceVersion: "16407" uid: 6d8b8e90-691c-463a-b733-4234b937a46b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:28:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16007" uid: bb7c2cb8-3e79-4e8e-9872-2d886e9c6eae kind: Event lastTimestamp: "2026-07-28T17:29:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:11Z" name: success-200-isvc-8c20d-predictor.18c684459dacabc1 namespace: kserve-ci-e2e-test resourceVersion: "16408" uid: 9ca9686c-658b-4bf2-85b0-5593614286d3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: bb7c2cb8-3e79-4e8e-9872-2d886e9c6eae kind: Event lastTimestamp: "2026-07-28T17:29:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:56Z" name: success-200-isvc-8c20d-predictor.18c6844c9b356a38 namespace: kserve-ci-e2e-test resourceVersion: "16802" uid: 9b6e30b4-cbaa-4b85-9cb0-2c6f6f3b8817 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:29:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-8c20d-predictor namespace: kserve-ci-e2e-test resourceVersion: "16294" uid: bb7c2cb8-3e79-4e8e-9872-2d886e9c6eae kind: Event lastTimestamp: "2026-07-28T17:29:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:56Z" name: success-200-isvc-8c20d-predictor.18c6844c9ba04dae namespace: kserve-ci-e2e-test resourceVersion: "16803" uid: 744abb32-fd27-4545-bd39-27f794a078a6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:28:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "15984" uid: 8f0bb31d-9688-414c-a395-853c841e9941 kind: Event lastTimestamp: "2026-07-28T17:29:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:29:29Z" name: success-200-isvc-8c20d.18c684421ec12d26 namespace: kserve-ci-e2e-test resourceVersion: "16552" uid: 7fd7f204-d67f-4d9a-8c80-45f3ba275ac9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:29:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-8c20d namespace: kserve-ci-e2e-test resourceVersion: "16550" uid: 8f0bb31d-9688-414c-a395-853c841e9941 kind: Event lastTimestamp: "2026-07-28T17:29:29Z" message: InferenceService [success-200-isvc-8c20d] is Ready metadata: creationTimestamp: "2026-07-28T17:29:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:29:29Z" name: success-200-isvc-8c20d.18c6844d22ab94ca namespace: kserve-ci-e2e-test resourceVersion: "16551" uid: 1cd97afe-7f81-4290-939a-de8b3fa6fb29 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:48:21.861052Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25708" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c0ed1-predictor-76c894557-m64nt to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554ded26977 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: e0d4adc9-bfbc-4a62-a6df-391da6d69081 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25718" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554f87d20da namespace: kserve-ci-e2e-test resourceVersion: "25781" uid: c63a8b12-8ce3-40a3-9ff7-015ef9306336 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554f9fb8bd5 namespace: kserve-ci-e2e-test resourceVersion: "25784" uid: 0de93caf-6f3d-46f5-ad7f-27705a0c2067 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554fd1ea500 namespace: kserve-ci-e2e-test resourceVersion: "25786" uid: 736fbd21-944a-490d-955a-ef7825d60198 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554fd9518e3 namespace: kserve-ci-e2e-test resourceVersion: "25787" uid: ceb66eb9-fe87-4326-90a1-3aaa53bae184 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68554fd9fe0d7 namespace: kserve-ci-e2e-test resourceVersion: "25789" uid: 5891e9e6-feea-4dd3-8c6b-d18c23d97a3c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68555012f9396 namespace: kserve-ci-e2e-test resourceVersion: "25790" uid: 778dbe1d-362a-4257-b8e3-5b98432718bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:48:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:22Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c6855501adff5d namespace: kserve-ci-e2e-test resourceVersion: "25791" uid: 9ace4728-03b7-44d3-aeae-2bfd1245ac98 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T17:49:08Z" message: 'Readiness probe failed: dial tcp 10.132.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:08Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c685550d9a0370 namespace: kserve-ci-e2e-test resourceVersion: "26510" uid: 422a0d55-b010-4d9e-bfd9-61cba095cd8d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T18:05:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T18:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:51Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c686493d74b087 namespace: kserve-ci-e2e-test resourceVersion: "33641" uid: 07217766-2dd0-4205-8fee-b9cf20de7257 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T18:05:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T18:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:51Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c686493d7547ab namespace: kserve-ci-e2e-test resourceVersion: "33642" uid: c2ca5fd2-bfa8-4d2b-8136-d08f6a2827f6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c0ed1-predictor-76c894557-m64nt namespace: kserve-ci-e2e-test resourceVersion: "25711" uid: d0ad12a3-5566-454f-9129-57fa4b3e5495 kind: Event lastTimestamp: "2026-07-28T18:05:53Z" message: 'Readiness probe failed: Get "https://10.132.0.38:8643/healthz": dial tcp 10.132.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T18:05:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:53Z" name: success-200-isvc-c0ed1-predictor-76c894557-m64nt.18c68649c28cab70 namespace: kserve-ci-e2e-test resourceVersion: "33705" uid: f9671a05-5a48-429b-9635-41bf8eb68c35 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c0ed1-predictor-76c894557 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: 8c3fad7c-0848-4553-b74d-3801eb0efa4b kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'Created pod: success-200-isvc-c0ed1-predictor-76c894557-m64nt' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c0ed1-predictor-76c894557.18c68554de607de4 namespace: kserve-ci-e2e-test resourceVersion: "25710" uid: 010171ba-932d-49c9-b161-0d1a3df66448 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25689" uid: acbe2b90-d6f2-44cb-9300-5d3f2bfb6da2 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Scaled up replica set success-200-isvc-c0ed1-predictor-76c894557 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c0ed1-predictor.18c68554dcab08aa namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: f5d6e422-56fe-4810-8de3-aa461c8f6f4c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 7b6a2d17-ff51-432e-8bed-a2113f853206 kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: success-200-isvc-c0ed1-predictor.18c685585b2c6438 namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: add00ce2-2821-4e63-bbe7-52ac54d0ba0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "25694" uid: 7b6a2d17-ff51-432e-8bed-a2113f853206 kind: Event lastTimestamp: "2026-07-28T17:48:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:48:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:48:36Z" name: success-200-isvc-c0ed1-predictor.18c685585ba29087 namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: f14dc7f3-0343-4752-8159-cafacdd4e4ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 7b6a2d17-ff51-432e-8bed-a2113f853206 kind: Event lastTimestamp: "2026-07-28T17:49:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:51Z" name: success-200-isvc-c0ed1-predictor.18c6855bdaa71e7f namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: dcbd6e74-255e-49bf-9da5-27d0f019b83a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:48:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c0ed1-predictor namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 7b6a2d17-ff51-432e-8bed-a2113f853206 kind: Event lastTimestamp: "2026-07-28T17:49:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:48:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:51Z" name: success-200-isvc-c0ed1-predictor.18c6855bdb1e7d41 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 7977164b-e531-4c7e-9938-5b117ea22606 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 622f2808-a0f7-4725-ab8b-555c54639c20 kind: Event lastTimestamp: "2026-07-28T17:49:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:18Z" name: success-200-isvc-c0ed1.18c68554dcb6c576 namespace: kserve-ci-e2e-test resourceVersion: "26606" uid: 6224e82d-4e07-4181-a901-20c40a52518a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 622f2808-a0f7-4725-ab8b-555c54639c20 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-c0ed1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c0ed1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c0ed1.18c68554dd5749b0 namespace: kserve-ci-e2e-test resourceVersion: "25703" uid: ad4c15b2-14b5-42d1-b953-53c23c5dd8cd reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "25662" uid: 622f2808-a0f7-4725-ab8b-555c54639c20 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c0ed1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c0ed1.18c68554dd57971f namespace: kserve-ci-e2e-test resourceVersion: "25706" uid: 909d5a1b-99a8-4aac-93e9-78641a8e2d64 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26602" uid: 622f2808-a0f7-4725-ab8b-555c54639c20 kind: Event lastTimestamp: "2026-07-28T17:49:18Z" message: InferenceService [success-200-isvc-c0ed1] is Ready metadata: creationTimestamp: "2026-07-28T17:49:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:18Z" name: success-200-isvc-c0ed1.18c685621dec3153 namespace: kserve-ci-e2e-test resourceVersion: "26604" uid: 206cc015-b3e9-4506-b233-cb9c281de79e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:38:11.063180Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20634" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6a86e93b5 namespace: kserve-ci-e2e-test resourceVersion: "20639" uid: f5963436-27cd-4811-87f8-29f9fd82405e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6c21e9464 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: 3c681733-e43b-4fe3-8340-3ff1a0bc06c4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6c3b94e1f namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: 20db46e3-e087-42f8-88d6-b8e31aefa56b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6c7c7cc86 namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: 09da160c-f62e-48d5-971b-4544752d6e9c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6c8687b24 namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: 3e6d9012-4558-42f5-a53f-739c5e0637b5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6c8734908 namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: d71131c3-8dbc-4127-b93b-0b618bb50adb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6cba60eb0 namespace: kserve-ci-e2e-test resourceVersion: "20696" uid: b37e101e-3538-40b7-b624-96ea5967f734 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6cc197811 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 9b7f7c12-165a-40e4-95f2-9fb9617b1dc7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T17:38:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:47:38Z" message: 'Readiness probe failed: dial tcp 10.132.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:38:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:38Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c684c6ef8d78e1 namespace: kserve-ci-e2e-test resourceVersion: "25156" uid: 3b823dc9-98d2-4563-93e2-d19f74ced298 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:47:35Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:35Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c6854a2e32f48e namespace: kserve-ci-e2e-test resourceVersion: "25040" uid: 90bb79cc-1c65-45db-b848-3a18ed5b5dfc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:47:35Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:47:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:35Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c6854a2e336d97 namespace: kserve-ci-e2e-test resourceVersion: "25041" uid: e822d00e-922a-4ff6-82ab-0f8c71759695 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:47:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk namespace: kserve-ci-e2e-test resourceVersion: "20635" uid: a377c3f4-672e-4217-9efd-cbd90e225034 kind: Event lastTimestamp: "2026-07-28T17:47:38Z" message: 'Readiness probe failed: Get "https://10.132.0.32:8643/healthz": dial tcp 10.132.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:47:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:47:38Z" name: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk.18c6854ab80fccc7 namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 62ce7f46-b94d-4a72-ac7a-1fb70e869f29 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c57a1-predictor-6674cb8b85 namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: f009b5b2-a379-422e-9ab1-8a2be3d22cc5 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Created pod: success-200-isvc-c57a1-predictor-6674cb8b85-xxzrk' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor-6674cb8b85.18c684c6a7f7e534 namespace: kserve-ci-e2e-test resourceVersion: "20636" uid: f6e8b540-ef8b-4e29-8268-7f39cebe5fa7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20618" uid: 9b37f5f4-562e-4b6f-94de-a6acd8e9d455 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: Scaled up replica set success-200-isvc-c57a1-predictor-6674cb8b85 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1-predictor.18c684c6a651d3bd namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 3c529938-5c76-4ef3-a697-5b130cf3643d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: d23d6439-e9c3-457a-a6a2-bd5846d4017e kind: Event lastTimestamp: "2026-07-28T17:38:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:41Z" name: success-200-isvc-c57a1-predictor.18c684ca24df8d00 namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: e3fc64e5-21d7-4e13-bc00-6296bca56ddb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:38:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20620" uid: d23d6439-e9c3-457a-a6a2-bd5846d4017e kind: Event lastTimestamp: "2026-07-28T17:38:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:38:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:38:41Z" name: success-200-isvc-c57a1-predictor.18c684ca253e9f46 namespace: kserve-ci-e2e-test resourceVersion: "20964" uid: 831f5b50-58a6-4069-87fb-516324805f4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: d23d6439-e9c3-457a-a6a2-bd5846d4017e kind: Event lastTimestamp: "2026-07-28T17:39:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:56Z" name: success-200-isvc-c57a1-predictor.18c684d1235148db namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 59183d25-a145-4bda-a64c-5d19efde88b8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:38:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c57a1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: d23d6439-e9c3-457a-a6a2-bd5846d4017e kind: Event lastTimestamp: "2026-07-28T17:39:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:38:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:56Z" name: success-200-isvc-c57a1-predictor.18c684d1239995c0 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: f1615ffa-9b97-412f-afe4-02201d7c6785 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 1150c2eb-9f8d-4486-a848-873b256bd009 kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:08Z" name: success-200-isvc-c57a1.18c684c6a6532e03 namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: d4980bf7-b68a-4dd4-bf3c-f925d532bd48 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 1150c2eb-9f8d-4486-a848-873b256bd009 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'Failed to update status for InferenceService "success-200-isvc-c57a1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c57a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1.18c684c6a6d1c710 namespace: kserve-ci-e2e-test resourceVersion: "20630" uid: fdbc93ab-6f1f-4f37-85c5-566346006a7f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:38:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: 1150c2eb-9f8d-4486-a848-873b256bd009 kind: Event lastTimestamp: "2026-07-28T17:38:11Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c57a1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:38:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:38:11Z" name: success-200-isvc-c57a1.18c684c6a6d22fd2 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 1c4f4943-ddbf-466a-9a8a-832d9003f7dc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c57a1 namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 1150c2eb-9f8d-4486-a848-873b256bd009 kind: Event lastTimestamp: "2026-07-28T17:39:08Z" message: InferenceService [success-200-isvc-c57a1] is Ready metadata: creationTimestamp: "2026-07-28T17:39:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:08Z" name: success-200-isvc-c57a1.18c684d3fed52cca namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 4e083042-7473-48a7-a934-742eef0dd5c6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:39:07.069135Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21296" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3b2a58af7 namespace: kserve-ci-e2e-test resourceVersion: "21304" uid: bd23ac15-2433-423c-ae74-eb76c1466e03 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21310" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3ce491c51 namespace: kserve-ci-e2e-test resourceVersion: "21344" uid: e9da0d0b-77f2-4ed2-80ac-dfa84a9177f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" already present on machine metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3cfa54410 namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: ea217774-1bdc-46fc-be92-303521b8496d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3d2c99776 namespace: kserve-ci-e2e-test resourceVersion: "21348" uid: c2291ff3-c61c-49c8-a622-c6d2a21f5781 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3d336b33e namespace: kserve-ci-e2e-test resourceVersion: "21349" uid: 3767d59b-096c-4a34-a6ef-92253061e584 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3d3411cbc namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 28b0394b-3f93-4c67-8799-60119ca14f0b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3d623114c namespace: kserve-ci-e2e-test resourceVersion: "21351" uid: 60c6414e-ac80-4c57-9529-f70e774d3b2b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d3d6a81d5f namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: bce9f53c-38e7-4c6d-8b3c-4c2100c73d07 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:39:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:39:45Z" message: 'Readiness probe failed: dial tcp 10.132.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:39:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:39:45Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c684d4442fffea namespace: kserve-ci-e2e-test resourceVersion: "21735" uid: 29240ece-fe44-459e-8cc2-016baf2caecb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c68554da2e51cf namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 60cc6eff-f76b-44d3-8d90-d079071c744f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:48:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: a3668a6c-7b0b-4814-9ced-5f29d4f88ad1 kind: Event lastTimestamp: "2026-07-28T17:48:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:48:21Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7.18c68554da2ecad6 namespace: kserve-ci-e2e-test resourceVersion: "25675" uid: 6eb232c1-5f30-45d5-a76a-cc74677cfdd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c7add-predictor-fb5c8d6b9 namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 71add491-650b-4020-8b14-cf7526819467 kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Created pod: success-200-isvc-c7add-predictor-fb5c8d6b9-gkcg7' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor-fb5c8d6b9.18c684d3b2368bc7 namespace: kserve-ci-e2e-test resourceVersion: "21299" uid: 8d6e4f72-abf3-426b-adfe-98cacb0baa79 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21273" uid: c8b944d9-f06f-409b-b8db-48f9f9e3a6da kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: Scaled up replica set success-200-isvc-c7add-predictor-fb5c8d6b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add-predictor.18c684d3b08f1028 namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 6ab94587-9be1-47b7-9797-9548f99b6876 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: cd7e19dd-dd39-4ad9-8263-98a5ca79dc5c kind: Event lastTimestamp: "2026-07-28T17:39:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:37Z" name: success-200-isvc-c7add-predictor.18c684d72f43ffa3 namespace: kserve-ci-e2e-test resourceVersion: "21662" uid: defa0d07-b797-41ea-a0c5-bd3a1f08f75b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:39:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21280" uid: cd7e19dd-dd39-4ad9-8263-98a5ca79dc5c kind: Event lastTimestamp: "2026-07-28T17:39:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:39:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:39:37Z" name: success-200-isvc-c7add-predictor.18c684d72fb3938e namespace: kserve-ci-e2e-test resourceVersion: "21663" uid: 60b727c0-4235-4d34-ba0b-d01865381335 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:39:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: cd7e19dd-dd39-4ad9-8263-98a5ca79dc5c kind: Event lastTimestamp: "2026-07-28T17:40:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:22Z" name: success-200-isvc-c7add-predictor.18c684de2d54282a namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 187c59d9-8397-4132-a039-1158146711a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T17:39:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7add-predictor namespace: kserve-ci-e2e-test resourceVersion: "21544" uid: cd7e19dd-dd39-4ad9-8263-98a5ca79dc5c kind: Event lastTimestamp: "2026-07-28T17:40:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:39:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:40:22Z" name: success-200-isvc-c7add-predictor.18c684de2d9bab73 namespace: kserve-ci-e2e-test resourceVersion: "22065" uid: 07dae2aa-7bdb-42a1-96bd-995a65f95b5e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 268471ee-3b62-47eb-aec2-a7788560e1ed kind: Event lastTimestamp: "2026-07-28T17:39:55Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:55Z" name: success-200-isvc-c7add.18c684d3b0a4fbfd namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: 930b260b-2b9f-4502-b2b7-49de85ee1634 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 268471ee-3b62-47eb-aec2-a7788560e1ed kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'Failed to update status for InferenceService "success-200-isvc-c7add": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7add": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add.18c684d3b10eaa41 namespace: kserve-ci-e2e-test resourceVersion: "21292" uid: e7a39751-28e4-4f32-a8ad-e15ec2e2c55e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21251" uid: 268471ee-3b62-47eb-aec2-a7788560e1ed kind: Event lastTimestamp: "2026-07-28T17:39:07Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c7add": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T17:39:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:07Z" name: success-200-isvc-c7add.18c684d3b10f0a2f namespace: kserve-ci-e2e-test resourceVersion: "21293" uid: a7a509c4-3eb9-4c02-9305-9a935a137917 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:39:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7add namespace: kserve-ci-e2e-test resourceVersion: "21831" uid: 268471ee-3b62-47eb-aec2-a7788560e1ed kind: Event lastTimestamp: "2026-07-28T17:39:55Z" message: InferenceService [success-200-isvc-c7add] is Ready metadata: creationTimestamp: "2026-07-28T17:39:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:39:55Z" name: success-200-isvc-c7add.18c684deff617181 namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: 40abb1dd-6dee-483a-b4d0-b9602d753192 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:26:11.766452Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp to ip-10-0-134-28.ec2.internal metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:26:11Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6841f2f018135 namespace: kserve-ci-e2e-test resourceVersion: "14264" uid: 4db8bb8d-da0c-435e-9a3e-084c6633806e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14268" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:26:12Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6841f487d17c6 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: b2023d04-1f6a-47d8-b1d1-8502e341f7a0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:12Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" metadata: creationTimestamp: "2026-07-28T17:26:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:12Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6841f49daa058 namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: f2a9a467-908e-4e02-8969-9d5fa24765a4 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:27Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1814" in 15.776s (15.776s including waiting). Image size: 1334926958 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c68422f62f0975 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: ae25a243-ca35-48b7-8a81-a49a711f7484 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c68422fa4b2eb7 namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: b65269cf-3083-40b2-b0b6-a35192a9e164 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c68422faafe310 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: db137240-ae95-457e-b2b6-8bc3707d2857 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T17:26:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:28Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c68422fabcde7a namespace: kserve-ci-e2e-test resourceVersion: "14562" uid: 40d04ae2-699a-4f41-a8fe-7ddad0f4e0d0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.587s (2.587s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6842394f4ac0e namespace: kserve-ci-e2e-test resourceVersion: "14578" uid: 86eb6b6f-6901-4874-a617-136cf791b3e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c684239892a55c namespace: kserve-ci-e2e-test resourceVersion: "14581" uid: 6e6625ab-e474-4e50-98d2-4249664d446e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:26:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:26:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:26:30Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6842398f92ea5 namespace: kserve-ci-e2e-test resourceVersion: "14583" uid: 29787aa2-c957-4f0c-9a05-b04b36021147 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T17:26:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:26:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c68423c8a7ee81 namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 95056b3e-af55-45fa-a31e-5e47ae466d21 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:45Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6843516dd6fe9 namespace: kserve-ci-e2e-test resourceVersion: "15332" uid: edb074c1-9863-4c79-8d87-6abef8a3a6a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:45Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c6843516de774e namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: d535a2cb-12a0-4c0d-931a-f860b7ebfde9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp namespace: kserve-ci-e2e-test resourceVersion: "14261" uid: 018dbbed-e2af-491e-8a6c-4b03eccbe7dd kind: Event lastTimestamp: "2026-07-28T17:27:47Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T17:27:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:47Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp.18c684357ad54db2 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 1cd8db8f-cf27-4480-802b-f2ef7cdf763a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-28.ec2.internal source: component: kubelet host: ip-10-0-134-28.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-cc402-predictor-5ffbc8f9cf namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: 0a0b79e5-cdc5-4350-8c1d-330562297b6a kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: 'Created pod: success-200-isvc-cc402-predictor-5ffbc8f9cf-2hxxp' metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: success-200-isvc-cc402-predictor-5ffbc8f9cf.18c6841f2e8da49e namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: b7bcaee0-29e6-4961-afac-2c7d7938d9a3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14249" uid: 2f517e2f-3abf-45bb-a72c-05231aee870b kind: Event lastTimestamp: "2026-07-28T17:26:11Z" message: Scaled up replica set success-200-isvc-cc402-predictor-5ffbc8f9cf from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:11Z" name: success-200-isvc-cc402-predictor.18c6841f2cecbca0 namespace: kserve-ci-e2e-test resourceVersion: "14252" uid: d44e7c78-cc34-482f-9e45-92da61853724 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: f842b433-a118-49ce-a45d-abadc9579136 kind: Event lastTimestamp: "2026-07-28T17:26:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:41Z" name: success-200-isvc-cc402-predictor.18c68422ab68057b namespace: kserve-ci-e2e-test resourceVersion: "14675" uid: 2ad0a20f-92bd-4d9e-afa6-a8b0b3138fb0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:26:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: f842b433-a118-49ce-a45d-abadc9579136 kind: Event lastTimestamp: "2026-07-28T17:26:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:26:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:26:41Z" name: success-200-isvc-cc402-predictor.18c68422abfd3946 namespace: kserve-ci-e2e-test resourceVersion: "14676" uid: 781422a5-d1f6-4766-b48e-35cd0d66111c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: f842b433-a118-49ce-a45d-abadc9579136 kind: Event lastTimestamp: "2026-07-28T17:27:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:41Z" name: success-200-isvc-cc402-predictor.18c68429a992e1ae namespace: kserve-ci-e2e-test resourceVersion: "15249" uid: 5dfe65bf-88f5-4f8b-912f-200d1f1015ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T17:26:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-cc402-predictor namespace: kserve-ci-e2e-test resourceVersion: "14537" uid: f842b433-a118-49ce-a45d-abadc9579136 kind: Event lastTimestamp: "2026-07-28T17:27:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:26:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:41Z" name: success-200-isvc-cc402-predictor.18c68429aa05cf89 namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 205ba1bb-e8ad-4d8d-aa3d-508e652d5b88 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T17:26:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc402 namespace: kserve-ci-e2e-test resourceVersion: "14238" uid: 790e3c82-8695-42cc-8cb2-faf9f417dd99 kind: Event lastTimestamp: "2026-07-28T17:27:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T17:26:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:17Z" name: success-200-isvc-cc402.18c6841f2ceed35c namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 2553bab5-8c16-4bf2-abdd-023fa3cd766b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-cc402 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 790e3c82-8695-42cc-8cb2-faf9f417dd99 kind: Event lastTimestamp: "2026-07-28T17:27:17Z" message: InferenceService [success-200-isvc-cc402] is Ready metadata: creationTimestamp: "2026-07-28T17:27:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:17Z" name: success-200-isvc-cc402.18c6842e87b3c0e8 namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: c797f5ec-42a8-4e30-a29d-139f85c0c437 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:28:45.967266Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-65026-6f5d6b7d79-snrnq to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:28:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:28:45Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c6844316172bd2 namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 703188ad-6d04-40c9-aad2-05641c77ecaa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:28:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-65026-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:46Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c68443212547b6 namespace: kserve-ci-e2e-test resourceVersion: "16184" uid: 66ad7b0d-704b-42d7-b3ee-8bf7ad20af3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:46Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:28:46Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:28:46Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c68443535f1e18 namespace: kserve-ci-e2e-test resourceVersion: "16195" uid: 433d8025-fdc2-4359-a995-38499f8e667a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65026} kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:28:47Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:47Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c6844354b0d7c0 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: ec1e3eff-3848-4d59-88ff-6022340fc027 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65026} kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:28:47Z" message: 'Created container: switch-graph-65026' metadata: creationTimestamp: "2026-07-28T17:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:47Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c684435807a7bc namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: e0876d55-86db-4dc3-af4c-d2b056cebea0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65026} kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:28:47Z" message: Started container switch-graph-65026 metadata: creationTimestamp: "2026-07-28T17:28:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:47Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c6844358786736 namespace: kserve-ci-e2e-test resourceVersion: "16200" uid: 2f49cc9f-53c2-4a72-a997-579f1c4914cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:37:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65026} kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:37:00Z" message: Stopping container switch-graph-65026 metadata: creationTimestamp: "2026-07-28T17:37:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:00Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c684b63d66a832 namespace: kserve-ci-e2e-test resourceVersion: "19692" uid: 4d23c08d-c794-45db-8a47-35782761c8bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:37:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-65026} kind: Pod name: switch-graph-65026-6f5d6b7d79-snrnq namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: 49e6f64e-1f2a-4932-bc45-c98ddc93a5e5 kind: Event lastTimestamp: "2026-07-28T17:37:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:37:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:37:29Z" name: switch-graph-65026-6f5d6b7d79-snrnq.18c684b710d16a3c namespace: kserve-ci-e2e-test resourceVersion: "20046" uid: 901d8e3b-4146-4ce6-9241-8c389db7aee8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-65026-6f5d6b7d79 namespace: kserve-ci-e2e-test resourceVersion: "16160" uid: cad4a26b-3c44-4dd1-9562-6d2b8104ef4a kind: Event lastTimestamp: "2026-07-28T17:28:45Z" message: 'Created pod: switch-graph-65026-6f5d6b7d79-snrnq' metadata: creationTimestamp: "2026-07-28T17:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:45Z" name: switch-graph-65026-6f5d6b7d79.18c6844315a74d94 namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: 397fc608-bb54-4533-9d42-3c4c2e06c90c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-65026 namespace: kserve-ci-e2e-test resourceVersion: "16158" uid: e78972e4-c26f-49f5-8cc6-e845533ff8d5 kind: Event lastTimestamp: "2026-07-28T17:28:45Z" message: Scaled up replica set switch-graph-65026-6f5d6b7d79 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:28:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:28:45Z" name: switch-graph-65026.18c6844314e84875 namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: 0cf3237b-2355-475a-8d8c-0b51957467ab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:28:46Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-65026 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: 9c6d4c9b-679d-4fc9-b20d-93b1194516d9 kind: Event lastTimestamp: "2026-07-28T17:28:46Z" message: InferenceGraph [switch-graph-65026] is Ready metadata: creationTimestamp: "2026-07-28T17:28:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:28:46Z" name: switch-graph-65026.18c684431970ce0e namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: f46fa477-935d-4988-87e0-a117beb64836 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65026 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: ae8b353e-a46e-4153-a22f-62eca0e647a7 kind: Event lastTimestamp: "2026-07-28T17:29:15Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:15Z" name: switch-graph-65026.18c6844693cd6bce namespace: kserve-ci-e2e-test resourceVersion: "16445" uid: bfbd01f5-37ed-4868-8817-4d5d1aa18bb8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T17:29:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-65026 namespace: kserve-ci-e2e-test resourceVersion: "16161" uid: ae8b353e-a46e-4153-a22f-62eca0e647a7 kind: Event lastTimestamp: "2026-07-28T17:29:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:29:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:29:15Z" name: switch-graph-65026.18c6844694379677 namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: 075d139d-e227-4bfe-ae1e-8ca8cd4f079a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:49:31.982294Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-c0ed1-695854fc69-pt6qc to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:49:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:49:31Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c68565325fac33 namespace: kserve-ci-e2e-test resourceVersion: "26729" uid: e6add8d2-6702-4237-9f01-28dc58eaea72 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T17:49:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-c0ed1-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:32Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c6856541b85ea8 namespace: kserve-ci-e2e-test resourceVersion: "26741" uid: 26773580-70fb-4da7-a4ba-637cbb99b9f1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26733" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T17:49:33Z" message: Add eth0 [10.133.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:49:33Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c685656f85cb31 namespace: kserve-ci-e2e-test resourceVersion: "26747" uid: 186426ea-fd67-4edf-9938-82a5300c710c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c0ed1} kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T17:49:33Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" already present on machine metadata: creationTimestamp: "2026-07-28T17:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:33Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c685657137e440 namespace: kserve-ci-e2e-test resourceVersion: "26749" uid: 4f99cb49-c323-4257-af60-1d7f8c9d016a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c0ed1} kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T17:49:33Z" message: 'Created container: switch-graph-c0ed1' metadata: creationTimestamp: "2026-07-28T17:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:33Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c6856574415ebb namespace: kserve-ci-e2e-test resourceVersion: "26750" uid: ae5315ec-3b6e-46db-9e09-db4083c25a14 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c0ed1} kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T17:49:33Z" message: Started container switch-graph-c0ed1 metadata: creationTimestamp: "2026-07-28T17:49:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:49:33Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c6856574f3640f namespace: kserve-ci-e2e-test resourceVersion: "26751" uid: 8f48d504-486a-471d-80fd-9c04a5b99399 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T18:05:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c0ed1} kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T18:05:51Z" message: Stopping container switch-graph-c0ed1 metadata: creationTimestamp: "2026-07-28T18:05:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:05:51Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c68649314dd094 namespace: kserve-ci-e2e-test resourceVersion: "33625" uid: 4f5d7a15-930e-4fb0-b677-6665e0136237 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T18:05:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-c0ed1} kind: Pod name: switch-graph-c0ed1-695854fc69-pt6qc namespace: kserve-ci-e2e-test resourceVersion: "26724" uid: b21040c9-0cbf-4956-8112-4823c63772b3 kind: Event lastTimestamp: "2026-07-28T18:06:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T18:05:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T18:06:19Z" name: switch-graph-c0ed1-695854fc69-pt6qc.18c68649ef817afe namespace: kserve-ci-e2e-test resourceVersion: "33898" uid: d1627120-0c64-4446-95ab-3d4a5aa36702 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-c0ed1-695854fc69 namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 5b62c0a8-54ed-47ea-a07f-6f66d48066d2 kind: Event lastTimestamp: "2026-07-28T17:49:31Z" message: 'Created pod: switch-graph-c0ed1-695854fc69-pt6qc' metadata: creationTimestamp: "2026-07-28T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:31Z" name: switch-graph-c0ed1-695854fc69.18c6856531df58e5 namespace: kserve-ci-e2e-test resourceVersion: "26725" uid: 62092757-397c-4905-b749-f4a3e4abdb13 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26717" uid: ab9c3abc-70ba-4007-b738-d5c856944931 kind: Event lastTimestamp: "2026-07-28T17:49:31Z" message: Scaled up replica set switch-graph-c0ed1-695854fc69 from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:31Z" name: switch-graph-c0ed1.18c6856530a38381 namespace: kserve-ci-e2e-test resourceVersion: "26720" uid: 2b56cdec-ccd5-4d11-8c24-a294d877b50f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26735" uid: 6f512d3a-87f7-4669-87c0-56c96562d44d kind: Event lastTimestamp: "2026-07-28T17:49:32Z" message: InferenceGraph [switch-graph-c0ed1] is Ready metadata: creationTimestamp: "2026-07-28T17:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:49:32Z" name: switch-graph-c0ed1.18c685653798aaeb namespace: kserve-ci-e2e-test resourceVersion: "26737" uid: 5f5bac1e-a78d-4584-b06d-ca3db57f162e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: 4ea74655-b2cd-4dfc-bc94-37bdf001090f kind: Event lastTimestamp: "2026-07-28T17:49:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:46Z" name: switch-graph-c0ed1.18c68568afce565e namespace: kserve-ci-e2e-test resourceVersion: "26886" uid: bb70e2da-6fba-424e-a80a-96f550911cb5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:49:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: 4ea74655-b2cd-4dfc-bc94-37bdf001090f kind: Event lastTimestamp: "2026-07-28T17:49:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T17:49:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:49:46Z" name: switch-graph-c0ed1.18c68568b06ca098 namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: f5925434-5552-478f-84fe-1680854fa90f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4ea74655-b2cd-4dfc-bc94-37bdf001090f kind: Event lastTimestamp: "2026-07-28T17:50:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:50:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:01Z" name: switch-graph-c0ed1.18c6856c2f54e75c namespace: kserve-ci-e2e-test resourceVersion: "27030" uid: 93e3b1ea-8b8c-46d5-aaba-c7476dded6e9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:50:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-c0ed1 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: 4ea74655-b2cd-4dfc-bc94-37bdf001090f kind: Event lastTimestamp: "2026-07-28T17:50:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T17:50:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:50:02Z" name: switch-graph-c0ed1.18c6856c2faade48 namespace: kserve-ci-e2e-test resourceVersion: "27032" uid: 4a0ba29c-9eaa-415b-96f1-f8d9ada98f26 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T17:27:31.564577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-cc402-d66f86ff-nhnxg to ip-10-0-136-215.ec2.internal metadata: creationTimestamp: "2026-07-28T17:27:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T17:27:31Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68431c3583e6e namespace: kserve-ci-e2e-test resourceVersion: "15144" uid: e9b83e77-796a-43cb-a6f2-1fb9c5572f83 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-84cbcfb569-vgtsb source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-cc402-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:31Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68431cb1f4fb0 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: dadfd98c-1cf9-4850-8364-7e906e17f3ba reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:32Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T17:27:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T17:27:32Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68432016f828c namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: db0baec0-c7cd-4505-a4cc-16368ec300f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:32Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" metadata: creationTimestamp: "2026-07-28T17:27:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:32Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c6843202c10dda namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: f161b2ea-42f3-4e6d-b5ea-a945378b7252 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:35Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:24a06dcfbbe7f0d25115c5a79181136502c6b1ca34878eb57638d7eb57f020b2" in 2.662s (2.662s including waiting). Image size: 218226814 bytes.' metadata: creationTimestamp: "2026-07-28T17:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:35Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68432a17b0d65 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 1c5508e4-ab47-4740-b275-03a422abb52c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:35Z" message: 'Created container: switch-graph-cc402' metadata: creationTimestamp: "2026-07-28T17:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:35Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68432a597c646 namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 7fd3b194-50f4-4a72-be0b-87769f1cb31d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:35Z" message: Started container switch-graph-cc402 metadata: creationTimestamp: "2026-07-28T17:27:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:35Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c68432a60c9b7c namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: c292d758-6ccb-462b-aee2-3213f8dbb264 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:27:45Z" message: Stopping container switch-graph-cc402 metadata: creationTimestamp: "2026-07-28T17:27:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:27:45Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c684350f1c1229 namespace: kserve-ci-e2e-test resourceVersion: "15305" uid: edb3a230-0216-43a1-b07d-251c2d1c70a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T17:27:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-cc402} kind: Pod name: switch-graph-cc402-d66f86ff-nhnxg namespace: kserve-ci-e2e-test resourceVersion: "15141" uid: 0a6d7387-3385-4965-86dd-77ff40a8f2bc kind: Event lastTimestamp: "2026-07-28T17:28:11Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T17:27:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T17:28:11Z" name: switch-graph-cc402-d66f86ff-nhnxg.18c684354e6f0902 namespace: kserve-ci-e2e-test resourceVersion: "15670" uid: 27f5fa83-d272-4c1c-9570-f3c4e559d7aa reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-136-215.ec2.internal source: component: kubelet host: ip-10-0-136-215.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-cc402-d66f86ff namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: 39c07950-7b62-47ab-b31c-eeab735964f2 kind: Event lastTimestamp: "2026-07-28T17:27:31Z" message: 'Created pod: switch-graph-cc402-d66f86ff-nhnxg' metadata: creationTimestamp: "2026-07-28T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:31Z" name: switch-graph-cc402-d66f86ff.18c68431c2e6acdd namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: 55b1cac9-4ed2-4b78-a541-5c90cb803253 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-cc402 namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: a3d8ff21-238d-4959-b844-2edf535346f2 kind: Event lastTimestamp: "2026-07-28T17:27:31Z" message: Scaled up replica set switch-graph-cc402-d66f86ff from 0 to 1 metadata: creationTimestamp: "2026-07-28T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T17:27:31Z" name: switch-graph-cc402.18c68431c227168c namespace: kserve-ci-e2e-test resourceVersion: "15137" uid: cf1be300-1488-4f98-8bf7-d4ccaa0b2778 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T17:27:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-cc402 namespace: kserve-ci-e2e-test resourceVersion: "15151" uid: 691e16ef-6f25-41db-a763-04fa19bb217d kind: Event lastTimestamp: "2026-07-28T17:27:31Z" message: InferenceGraph [switch-graph-cc402] is Ready metadata: creationTimestamp: "2026-07-28T17:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T17:27:31Z" name: switch-graph-cc402.18c68431ca2571d1 namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: 765549f1-2896-4dc4-9483-32ed19ab372f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33899"