--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:57:56.020806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-75e21-5bdd974b9c-4h42x to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:57:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:57:56Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757b99cdd438 namespace: kserve-ci-e2e-test resourceVersion: "21381" uid: 110ea258-3621-484b-b0ec-fe335205c9df reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:56Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-75e21-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:57:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:56Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757ba512fb5c namespace: kserve-ci-e2e-test resourceVersion: "21394" uid: d92e4435-1440-4fb3-b0bc-5431e6ee5f52 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:57:57Z" message: Add eth0 [10.132.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:57:57Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757bd7411b55 namespace: kserve-ci-e2e-test resourceVersion: "21401" uid: 3b233fed-7301-4d2d-87eb-9ebba9bb110a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75e21} kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:57:57Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:57Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757bd8bfcb36 namespace: kserve-ci-e2e-test resourceVersion: "21403" uid: e46c6735-f264-4363-b654-a78ead6781e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75e21} kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:57:57Z" message: 'Created container: ensemble-graph-75e21' metadata: creationTimestamp: "2026-07-28T12:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:57Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757bdb7f274d namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: d8d5f732-a97d-494c-9898-995979a9e8a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75e21} kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:57:57Z" message: Started container ensemble-graph-75e21 metadata: creationTimestamp: "2026-07-28T12:57:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:57Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757bdc030041 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: edd1c525-7f5d-4697-a2d4-ece5a8011739 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75e21} kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Stopping container ensemble-graph-75e21 metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757df3081650 namespace: kserve-ci-e2e-test resourceVersion: "21504" uid: e7df2376-e58f-4fc4-8f8e-f488101170bd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-75e21} kind: Pod name: ensemble-graph-75e21-5bdd974b9c-4h42x namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: c92af120-de57-4b4d-8420-088863f54fe2 kind: Event lastTimestamp: "2026-07-28T12:58:33Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:33Z" name: ensemble-graph-75e21-5bdd974b9c-4h42x.18c6757e7a3e1097 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: b1794b52-4933-44f1-9f2f-fa7d6aab362b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-75e21-5bdd974b9c namespace: kserve-ci-e2e-test resourceVersion: "21372" uid: 5d3ff32c-1726-4de8-bdef-cb59c80e3ffc kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: 'Created pod: ensemble-graph-75e21-5bdd974b9c-4h42x' metadata: creationTimestamp: "2026-07-28T12:57:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:56Z" name: ensemble-graph-75e21-5bdd974b9c.18c6757b99667b16 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: 5ef8d003-a08d-44ec-8819-306762ca1632 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-75e21 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: 48d2e432-fce5-462c-aa7e-5e9f728dc2b3 kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: Scaled up replica set ensemble-graph-75e21-5bdd974b9c from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:57:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:56Z" name: ensemble-graph-75e21.18c6757b9891d68e namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: b4d71ec4-3ede-4aa6-8455-7e3318faae7e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:56Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-75e21 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 8ffd469b-6ee0-4999-bd48-5c67767be297 kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: InferenceGraph [ensemble-graph-75e21] is Ready metadata: creationTimestamp: "2026-07-28T12:57:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:56Z" name: ensemble-graph-75e21.18c6757b9d141a8e namespace: kserve-ci-e2e-test resourceVersion: "21390" uid: ac009853-c554-4e93-82cc-118746c9b9be reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:59:06.328436Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22349" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-da172-54c5896848-8vfjc to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c6758bf8772277 namespace: kserve-ci-e2e-test resourceVersion: "22354" uid: 5e79c38d-b457-4ea6-ade8-82dc52f1ea15 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22358" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: Add eth0 [10.132.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c6758c12c82e49 namespace: kserve-ci-e2e-test resourceVersion: "22373" uid: daff19cb-c575-4fd5-93c5-055bf7ce3957 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da172} kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c6758c14180e02 namespace: kserve-ci-e2e-test resourceVersion: "22375" uid: 43ca10a9-13e8-40fa-bb62-88d318a9097d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da172} kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: 'Created container: ensemble-graph-da172' metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c6758c17099ee5 namespace: kserve-ci-e2e-test resourceVersion: "22376" uid: 7c66a3aa-a3ea-4539-9939-5c1148e8c1db reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da172} kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: Started container ensemble-graph-da172 metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c6758c17c4aa75 namespace: kserve-ci-e2e-test resourceVersion: "22377" uid: a4bae3a3-96f7-46d8-bd0e-63c7fa725f52 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da172} kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Stopping container ensemble-graph-da172 metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c675ff2565b918 namespace: kserve-ci-e2e-test resourceVersion: "26177" uid: 6ecce420-6d9e-45be-8a15-a3fe4950a43e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:07:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-da172} kind: Pod name: ensemble-graph-da172-54c5896848-8vfjc namespace: kserve-ci-e2e-test resourceVersion: "22351" uid: 2c019bab-57fc-45cf-bc44-48d81238c31c kind: Event lastTimestamp: "2026-07-28T13:07:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T13:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:48Z" name: ensemble-graph-da172-54c5896848-8vfjc.18c675ffc75954ef namespace: kserve-ci-e2e-test resourceVersion: "26515" uid: 1794e2fa-51dc-4c93-a189-7a2d6f713ff3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-da172-54c5896848 namespace: kserve-ci-e2e-test resourceVersion: "22346" uid: 41e8b753-4c0c-4a26-b93c-93c4d708a447 kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: 'Created pod: ensemble-graph-da172-54c5896848-8vfjc' metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172-54c5896848.18c6758bf7e826e9 namespace: kserve-ci-e2e-test resourceVersion: "22350" uid: 4e3b65a3-75c5-4113-936d-b76da5d5e3c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-da172 namespace: kserve-ci-e2e-test resourceVersion: "22342" uid: 86248367-57ff-4b3d-b95b-16919a822b61 kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: Scaled up replica set ensemble-graph-da172-54c5896848 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172.18c6758bf72237d6 namespace: kserve-ci-e2e-test resourceVersion: "22347" uid: 7f526b84-a543-4ec9-9bff-7175edfbb7bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-da172 namespace: kserve-ci-e2e-test resourceVersion: "22360" uid: bab3d883-f9b8-4dc9-b63d-1dc5f64d0503 kind: Event lastTimestamp: "2026-07-28T12:59:06Z" message: InferenceGraph [ensemble-graph-da172] is Ready metadata: creationTimestamp: "2026-07-28T12:59:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:59:06Z" name: ensemble-graph-da172.18c6758bfbba0e3b namespace: kserve-ci-e2e-test resourceVersion: "22362" uid: 7f457c51-46e3-419b-918c-630f61a5bafb reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da172 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: c09242b1-13bc-476c-b0b5-fa8155a9a29f kind: Event lastTimestamp: "2026-07-28T12:59:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:36Z" name: ensemble-graph-da172.18c6758f75774cee namespace: kserve-ci-e2e-test resourceVersion: "22622" uid: c5d07a9f-9a67-4073-8190-23dac738cd96 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:59:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-da172 namespace: kserve-ci-e2e-test resourceVersion: "22344" uid: c09242b1-13bc-476c-b0b5-fa8155a9a29f kind: Event lastTimestamp: "2026-07-28T12:59:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:36Z" name: ensemble-graph-da172.18c6758f75e41af7 namespace: kserve-ci-e2e-test resourceVersion: "22623" uid: 60f7e299-fc76-4dfe-9948-7dfcd9582322 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:08:31.564359Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27145" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-13bf1-predictor-bbffd5856-2kmmm to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760f9326e857 namespace: kserve-ci-e2e-test resourceVersion: "27149" uid: 781c35c8-7554-44ce-8502-ee38006dec40 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27159" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Add eth0 [10.132.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760facdcc5c1 namespace: kserve-ci-e2e-test resourceVersion: "27172" uid: d36bd5c6-a617-434b-9112-5a9ec8334a10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fae3b360f namespace: kserve-ci-e2e-test resourceVersion: "27176" uid: 971c3c71-8909-4561-a16e-e472a40757c2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb17207f1 namespace: kserve-ci-e2e-test resourceVersion: "27180" uid: aabba0ed-d030-4295-8dea-9c76f651916b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb1f44e3d namespace: kserve-ci-e2e-test resourceVersion: "27181" uid: fc39949a-337c-4f26-9255-0d7fb67856cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb1fe7c94 namespace: kserve-ci-e2e-test resourceVersion: "27182" uid: 8249bd87-7c86-42d2-bf15-3fb200806520 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb4ba9edd namespace: kserve-ci-e2e-test resourceVersion: "27183" uid: 275acb92-6cb9-470e-84fb-5067c48887a0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb5307bab namespace: kserve-ci-e2e-test resourceVersion: "27184" uid: 97608071-8879-41b9-b70d-4203a69e79ae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:17:48Z" message: 'Readiness probe failed: dial tcp 10.132.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:48Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c6760fb9c8eb84 namespace: kserve-ci-e2e-test resourceVersion: "31266" uid: 2b879362-c48a-4445-b747-4cc4a72666ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:17:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:46Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c67690c4193dc9 namespace: kserve-ci-e2e-test resourceVersion: "31248" uid: 65062809-d217-400c-844d-66dbd32a28cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:17:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:46Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c67690c419ff32 namespace: kserve-ci-e2e-test resourceVersion: "31249" uid: be08024b-7c06-4900-bd93-d8a7b80ab6c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm namespace: kserve-ci-e2e-test resourceVersion: "27147" uid: 32396bba-6723-4e61-a74c-16ee1b50628b kind: Event lastTimestamp: "2026-07-28T13:17:48Z" message: 'Readiness probe failed: Get "https://10.132.0.46:8643/healthz": dial tcp 10.132.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:48Z" name: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm.18c676912e55cad4 namespace: kserve-ci-e2e-test resourceVersion: "31264" uid: e45e50c4-8fbe-474e-92d4-cdce4e54e1eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-13bf1-predictor-bbffd5856 namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: dbb77ae3-defa-438a-bf76-867934d72472 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: 'Created pod: error-404-isvc-13bf1-predictor-bbffd5856-2kmmm' metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1-predictor-bbffd5856.18c6760f929bbcb7 namespace: kserve-ci-e2e-test resourceVersion: "27146" uid: 0be2b0b2-794f-497a-b3ff-b394faec683c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27133" uid: 38c1dade-69c7-4e3e-b5cf-4fdd6be0f0a6 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Scaled up replica set error-404-isvc-13bf1-predictor-bbffd5856 from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1-predictor.18c6760f91060483 namespace: kserve-ci-e2e-test resourceVersion: "27137" uid: 65d9dab5-085c-4093-b982-e3e52b1c8101 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 1f02ade0-dac1-4ec1-a312-04cdb1f23c25 kind: Event lastTimestamp: "2026-07-28T13:09:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:01Z" name: error-404-isvc-13bf1-predictor.18c676130f86b2da namespace: kserve-ci-e2e-test resourceVersion: "27445" uid: e91516a2-e5b0-457a-8469-f329557a26c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 1f02ade0-dac1-4ec1-a312-04cdb1f23c25 kind: Event lastTimestamp: "2026-07-28T13:09:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:01Z" name: error-404-isvc-13bf1-predictor.18c67613100877d0 namespace: kserve-ci-e2e-test resourceVersion: "27446" uid: 94892886-952c-4143-a069-f1fd02e5c98e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 1f02ade0-dac1-4ec1-a312-04cdb1f23c25 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: error-404-isvc-13bf1-predictor.18c6761a0d4f48f4 namespace: kserve-ci-e2e-test resourceVersion: "27903" uid: ddf48a1f-5f4a-4412-bdda-d2622270324d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27322" uid: 1f02ade0-dac1-4ec1-a312-04cdb1f23c25 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: error-404-isvc-13bf1-predictor.18c6761a0db162f2 namespace: kserve-ci-e2e-test resourceVersion: "27904" uid: 9c03a884-4e03-4216-8765-726e2ada56e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: f12245b7-a70f-47e7-9947-8d7187c706e6 kind: Event lastTimestamp: "2026-07-28T13:09:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:18Z" name: error-404-isvc-13bf1.18c6760f9104c81d namespace: kserve-ci-e2e-test resourceVersion: "27605" uid: 7439305e-2a04-40cc-b26d-155bcdceea5e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: f12245b7-a70f-47e7-9947-8d7187c706e6 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: 'Failed to update status for InferenceService "error-404-isvc-13bf1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-13bf1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1.18c6760f919249ff namespace: kserve-ci-e2e-test resourceVersion: "27142" uid: 0ea96a1b-6228-4a4d-98e2-ad58171ca788 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27117" uid: f12245b7-a70f-47e7-9947-8d7187c706e6 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-13bf1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-13bf1.18c6760f9192e924 namespace: kserve-ci-e2e-test resourceVersion: "27143" uid: 31dc6611-6b2d-44af-88e2-719b977b6b7c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27603" uid: f12245b7-a70f-47e7-9947-8d7187c706e6 kind: Event lastTimestamp: "2026-07-28T13:09:18Z" message: InferenceService [error-404-isvc-13bf1] is Ready metadata: creationTimestamp: "2026-07-28T13:09:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:18Z" name: error-404-isvc-13bf1.18c6761a745b1908 namespace: kserve-ci-e2e-test resourceVersion: "27604" uid: 354ff942-9951-4ebe-afc7-3f9ff69bd692 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:07:21.247469Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26241" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff33f04212 namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: dff5bc98-528c-491d-9742-93d8c3b3bdaf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Add eth0 [10.132.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff4d93a4c2 namespace: kserve-ci-e2e-test resourceVersion: "26282" uid: b800a31b-ab73-4a30-87da-99c39d5c812f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff4ee458a3 namespace: kserve-ci-e2e-test resourceVersion: "26285" uid: 8e4c6c52-fff6-48e2-8bc4-d134f10ba451 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff5266d789 namespace: kserve-ci-e2e-test resourceVersion: "26286" uid: f0d4a155-232e-473f-9f25-1963c1c9fa40 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff52e273aa namespace: kserve-ci-e2e-test resourceVersion: "26287" uid: 238f8a92-c6f7-4c67-aa18-f01914eab646 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff52ebdaed namespace: kserve-ci-e2e-test resourceVersion: "26288" uid: a5fd8e17-ba23-4070-a963-ce9b34d1a511 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff55a9d9b9 namespace: kserve-ci-e2e-test resourceVersion: "26289" uid: 010705a0-a8e2-4447-b487-6e10f62483a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff561eda4b namespace: kserve-ci-e2e-test resourceVersion: "26290" uid: fa4f55a1-c4ef-42e9-ad38-872274a2bd84 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:07:57Z" message: 'Readiness probe failed: dial tcp 10.132.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:57Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c675ff592b568a namespace: kserve-ci-e2e-test resourceVersion: "26597" uid: fb4b6772-d3dc-44aa-b87e-1df75c823641 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c6760f8d0bf8c0 namespace: kserve-ci-e2e-test resourceVersion: "27114" uid: 72c7e11b-5b88-46db-a7a2-1834adc7636e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c6760f8d0f48f2 namespace: kserve-ci-e2e-test resourceVersion: "27120" uid: 7b7c0eb1-925a-419d-ab7d-bce6bd558556 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj namespace: kserve-ci-e2e-test resourceVersion: "26243" uid: 9b966603-3307-401a-ae83-d8ae74c7c7b1 kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: 'Readiness probe failed: Get "https://10.132.0.44:8643/healthz": dial tcp 10.132.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj.18c6760fe16ae08a namespace: kserve-ci-e2e-test resourceVersion: "27195" uid: eab2660f-a7e5-41c1-9241-e67fae673820 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4b3a0-predictor-5dc499d786 namespace: kserve-ci-e2e-test resourceVersion: "26234" uid: 986406de-3499-44c5-bf4f-5d97bf13c47a kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Created pod: error-404-isvc-4b3a0-predictor-5dc499d786-7lnlj' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor-5dc499d786.18c675ff3382aed3 namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 821cdd5b-fc2d-4376-95d4-1b78bd27a6ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26230" uid: aa80829b-3627-4aea-893b-868398333ff4 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Scaled up replica set error-404-isvc-4b3a0-predictor-5dc499d786 from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0-predictor.18c675ff31d23ac7 namespace: kserve-ci-e2e-test resourceVersion: "26235" uid: 92bf1772-c7b9-4883-8693-c1370195c709 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: e8da39a1-85ed-4ba1-9da0-9bb9efb99ee7 kind: Event lastTimestamp: "2026-07-28T13:07:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:51Z" name: error-404-isvc-4b3a0-predictor.18c67602b03ade93 namespace: kserve-ci-e2e-test resourceVersion: "26541" uid: 6e9622e9-4c61-4e59-b508-970f444a39b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26233" uid: e8da39a1-85ed-4ba1-9da0-9bb9efb99ee7 kind: Event lastTimestamp: "2026-07-28T13:07:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:51Z" name: error-404-isvc-4b3a0-predictor.18c67602b0a311a3 namespace: kserve-ci-e2e-test resourceVersion: "26542" uid: 38e8ded6-8205-4973-87ba-c3a5d3c8cf61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: e8da39a1-85ed-4ba1-9da0-9bb9efb99ee7 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-4b3a0-predictor.18c67609ae0a41ce namespace: kserve-ci-e2e-test resourceVersion: "26921" uid: dd9e4165-7dee-4ac7-8ed7-c8b50cbc411f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26427" uid: e8da39a1-85ed-4ba1-9da0-9bb9efb99ee7 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:08:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-4b3a0-predictor.18c67609ae7fefe8 namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 84030c63-a8bf-42d9-be13-1a72d99d025f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 2aa8ab8e-9214-44de-9fdd-3cde1c59f95d kind: Event lastTimestamp: "2026-07-28T13:08:07Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:07Z" name: error-404-isvc-4b3a0.18c675ff31d21991 namespace: kserve-ci-e2e-test resourceVersion: "26677" uid: 0680efbf-beb5-4381-a4d3-09712ece09d9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 2aa8ab8e-9214-44de-9fdd-3cde1c59f95d kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Failed to update status for InferenceService "error-404-isvc-4b3a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b3a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0.18c675ff32629518 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: 76b9d973-fb9f-4bcf-8ee4-4e750658b8dc reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26215" uid: 2aa8ab8e-9214-44de-9fdd-3cde1c59f95d kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-4b3a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-4b3a0.18c675ff3262e89c namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: 5d0099ec-3009-4eb0-b677-5993af03b8a8 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26674" uid: 2aa8ab8e-9214-44de-9fdd-3cde1c59f95d kind: Event lastTimestamp: "2026-07-28T13:08:07Z" message: InferenceService [error-404-isvc-4b3a0] is Ready metadata: creationTimestamp: "2026-07-28T13:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:07Z" name: error-404-isvc-4b3a0.18c6760a140b858e namespace: kserve-ci-e2e-test resourceVersion: "26675" uid: e6699db3-4dd3-4012-b814-4f66b366d5b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:56:56.040691Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20682" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-75e21-predictor-64646f5b4b-6j67r to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756da2b70286 namespace: kserve-ci-e2e-test resourceVersion: "20688" uid: e952ea10-e789-4356-951d-597fd587bfca reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-75e21-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756db359f048 namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: f3dd849e-f6d6-4cd5-922f-5421ff730772 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20699" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Add eth0 [10.132.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de0ba1e9e namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: d60ee5d5-2baa-4f6c-9d2d-d2509dccf806 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de2286679 namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 7942a4c5-1438-4df1-8c69-894e2031a76a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de52059ad namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: ca9c52e3-3cb3-494f-b3e2-0bb596f757b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de59190fb namespace: kserve-ci-e2e-test resourceVersion: "20730" uid: 36c7be98-9535-411b-99d3-0a2cb50f300d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de59b4cbb namespace: kserve-ci-e2e-test resourceVersion: "20731" uid: b047346b-963b-4b58-8783-8334b976daf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de956988b namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: bb96225e-cf0d-4d02-b270-e798709a3a57 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756de9d03ab1 namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: ec29ecd9-bbd6-4968-b284-43a0a5d232cb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:56:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:57:35Z" message: 'Readiness probe failed: dial tcp 10.132.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:56:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:35Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6756e57f93080 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 86ca55ae-7c62-4ef8-a5e5-c6eae529c978 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6757dfb952132 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: f8007139-a3ae-425c-a95d-84d01f2fbbd6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r namespace: kserve-ci-e2e-test resourceVersion: "20684" uid: 9ef29225-a748-4be2-8e05-af974a483507 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-75e21-predictor-64646f5b4b-6j67r.18c6757dfb95a26e namespace: kserve-ci-e2e-test resourceVersion: "21556" uid: 2223fe85-f1b9-4e6f-a09f-237c165596cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-75e21-predictor-64646f5b4b namespace: kserve-ci-e2e-test resourceVersion: "20664" uid: 3e16aadd-7858-44e8-89b3-bb26496c0907 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: 'Created pod: error-404-isvc-75e21-predictor-64646f5b4b-6j67r' metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-75e21-predictor-64646f5b4b.18c6756da248d668 namespace: kserve-ci-e2e-test resourceVersion: "20685" uid: df62764f-814c-489e-b8b3-510134dd1d16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20658" uid: 3714495c-17af-4844-8845-b08c8bb33ae8 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: Scaled up replica set error-404-isvc-75e21-predictor-64646f5b4b from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-75e21-predictor.18c6756da058b0b8 namespace: kserve-ci-e2e-test resourceVersion: "20670" uid: 12bf7a20-ecf4-46d5-9fd2-dd63346e8a55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 67635226-2807-49c3-8234-55f0c2d2ad87 kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:56Z" name: error-404-isvc-75e21-predictor.18c675711f40cc86 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: b9d8177d-034e-4aa0-bcb2-da79c9ea04d6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:57:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20663" uid: 67635226-2807-49c3-8234-55f0c2d2ad87 kind: Event lastTimestamp: "2026-07-28T12:57:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:57:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:56Z" name: error-404-isvc-75e21-predictor.18c675711fe90efb namespace: kserve-ci-e2e-test resourceVersion: "21391" uid: b7fe9b54-e75a-4347-9921-02d3ceb524ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "20648" uid: a56e7284-415e-4fa4-868c-5e93e22d00ca kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-75e21.18c6756da0795c1d namespace: kserve-ci-e2e-test resourceVersion: "21127" uid: c1b0dcdd-01ba-4fa8-95c0-8f64c4fe7809 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "21125" uid: a56e7284-415e-4fa4-868c-5e93e22d00ca kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: InferenceService [error-404-isvc-75e21] is Ready metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-75e21.18c67579131795cb namespace: kserve-ci-e2e-test resourceVersion: "21126" uid: 436e67cd-d626-4eee-9361-69362a601d73 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:46:00.854628Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9baeb-predictor-85d6f87596-8qwgd to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:46:00Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d516940584 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: ed6f2ee2-541c-44fd-8019-3bcd65667e30 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9baeb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d51f598afd namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: dc626d0d-1a82-4657-b915-33f44d9875e5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:46:01Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d553eb1d90 namespace: kserve-ci-e2e-test resourceVersion: "15254" uid: 6dce6f5f-2097-4e7d-8871-585f1f6aa1b6 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d555673cba namespace: kserve-ci-e2e-test resourceVersion: "15256" uid: 57d7eac4-1eae-4a6c-a719-5d08038d415d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:13Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" in 11.247s (11.247s including waiting). Image size: 1333902957 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:13Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d7f3d48286 namespace: kserve-ci-e2e-test resourceVersion: "15382" uid: 233a18c6-62c3-482c-973c-83131464f81d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:13Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d7f6e79469 namespace: kserve-ci-e2e-test resourceVersion: "15384" uid: baad6f1b-614d-48da-88d7-0da8063645dc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:13Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d7f76022c5 namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: 53a54dbc-dcd6-4201-bc5e-35e0591171db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T12:46:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:13Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d7f76fd77b namespace: kserve-ci-e2e-test resourceVersion: "15386" uid: 7e3697d5-3564-4d4c-967f-e72bc9da01ed reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.163s (2.163s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:15Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d8785f6ab7 namespace: kserve-ci-e2e-test resourceVersion: "15402" uid: b3c1f2f4-cce8-44e9-b2f3-b59a89f2c48e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:15Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d87cf81e10 namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: aa267556-ce44-439d-95fd-31b2e660886e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:46:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:15Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d87dce852a namespace: kserve-ci-e2e-test resourceVersion: "15404" uid: f4668fb8-7725-402b-9ff3-909d444c2aa1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:46:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: 'Readiness probe failed: dial tcp 10.132.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674d8ac026186 namespace: kserve-ci-e2e-test resourceVersion: "16325" uid: 4f089584-8193-4093-a070-89b85b483e61 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674ec78cb7d87 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 1def48db-6e79-4ca9-834b-f423c15ff648 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674ec78cc0331 namespace: kserve-ci-e2e-test resourceVersion: "16296" uid: 5fae79ae-e53a-42a4-926f-9e59ae1c68ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd namespace: kserve-ci-e2e-test resourceVersion: "15159" uid: 570b012f-e5f0-4b4e-b7a8-b9de5fd5cb73 kind: Event lastTimestamp: "2026-07-28T12:47:47Z" message: 'Readiness probe failed: Get "https://10.132.0.29:8643/healthz": dial tcp 10.132.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:47Z" name: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd.18c674ecb23f5253 namespace: kserve-ci-e2e-test resourceVersion: "16406" uid: 03c1408e-e3ba-4fc6-8093-5b09f8a350b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9baeb-predictor-85d6f87596 namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: 9c350640-513c-486e-8d25-193c4a62995b kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: 'Created pod: error-404-isvc-9baeb-predictor-85d6f87596-8qwgd' metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: error-404-isvc-9baeb-predictor-85d6f87596.18c674d5163a68c9 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: ad06421a-6133-47e0-a9ed-3bce7017ffc9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15145" uid: 049ee14b-00c2-4015-987e-38f9d1b71910 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Scaled up replica set error-404-isvc-9baeb-predictor-85d6f87596 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: error-404-isvc-9baeb-predictor.18c674d51358874b namespace: kserve-ci-e2e-test resourceVersion: "15150" uid: 379e8219-e565-40c0-ab63-36d105efc957 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 4283e120-adf9-41f5-bc4a-ca1359536cb0 kind: Event lastTimestamp: "2026-07-28T12:46:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:45Z" name: error-404-isvc-9baeb-predictor.18c674d891f17532 namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: d855cf56-0b92-42a7-b4ea-49502d9495cd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15147" uid: 4283e120-adf9-41f5-bc4a-ca1359536cb0 kind: Event lastTimestamp: "2026-07-28T12:46:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:45Z" name: error-404-isvc-9baeb-predictor.18c674d892562334 namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: cc739511-3494-434d-93c6-9f8ed8d34ea8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 4283e120-adf9-41f5-bc4a-ca1359536cb0 kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: error-404-isvc-9baeb-predictor.18c674dc112e0756 namespace: kserve-ci-e2e-test resourceVersion: "16102" uid: 62e0adda-e6eb-47fa-b2db-d39a72f107f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15414" uid: 4283e120-adf9-41f5-bc4a-ca1359536cb0 kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:46:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: error-404-isvc-9baeb-predictor.18c674dc119311c5 namespace: kserve-ci-e2e-test resourceVersion: "16103" uid: 5374409d-42bc-4248-af2d-76e37a65b153 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9baeb namespace: kserve-ci-e2e-test resourceVersion: "15131" uid: 589cc4fe-fd9c-4c4b-99ce-89dfd6cb49b4 kind: Event lastTimestamp: "2026-07-28T12:47:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:02Z" name: error-404-isvc-9baeb.18c674d5138779de namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: e7c1e46f-0cc2-4364-9b44-b67aed1ba9e4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9baeb namespace: kserve-ci-e2e-test resourceVersion: "15845" uid: 589cc4fe-fd9c-4c4b-99ce-89dfd6cb49b4 kind: Event lastTimestamp: "2026-07-28T12:47:02Z" message: InferenceService [error-404-isvc-9baeb] is Ready metadata: creationTimestamp: "2026-07-28T12:47:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:02Z" name: error-404-isvc-9baeb.18c674e369181992 namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: a85b575a-1ffb-4636-9aaf-34e8add662ed reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:58:56.154996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22204" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c675899a14c9b5 namespace: kserve-ci-e2e-test resourceVersion: "22208" uid: 2593dc64-e358-4998-af44-33a3df2437f2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22214" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Add eth0 [10.132.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589b3ba0ce0 namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: 269a8d07-e5c9-473a-9484-2b0ff3361962 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589b516bdd0 namespace: kserve-ci-e2e-test resourceVersion: "22229" uid: 4ea3386a-467b-4486-803c-bda9c59672f2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589b819c406 namespace: kserve-ci-e2e-test resourceVersion: "22230" uid: 63a20d5a-a6ae-469c-b9be-2b5b9af38ac0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589b8b6fb69 namespace: kserve-ci-e2e-test resourceVersion: "22231" uid: 72507325-2816-4b12-888f-37cc9ec67d80 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589b8c1422e namespace: kserve-ci-e2e-test resourceVersion: "22232" uid: 3b310156-850d-44db-ab5f-b77e10ce1be3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589bbd9250a namespace: kserve-ci-e2e-test resourceVersion: "22233" uid: 48693617-b7b3-415a-b49b-cbb85d96aa35 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c67589bc4e8021 namespace: kserve-ci-e2e-test resourceVersion: "22234" uid: a48d789b-8478-4b78-b9d8-6222ee156c23 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T12:59:34Z" message: 'Readiness probe failed: dial tcp 10.132.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:59:34Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c6758a2fc360f6 namespace: kserve-ci-e2e-test resourceVersion: "22608" uid: 8a6bd191-6656-4f5e-b42a-c204f2ed4f10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:20Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c6760d1106c0b6 namespace: kserve-ci-e2e-test resourceVersion: "26807" uid: 24255505-9e6e-4b59-9d82-4bdf1c3853c6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2 namespace: kserve-ci-e2e-test resourceVersion: "22205" uid: fcf6dd94-3518-4850-8caf-1cfab68c8671 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:20Z" name: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2.18c6760d110746fb namespace: kserve-ci-e2e-test resourceVersion: "26810" uid: 3a79afd2-e00a-4a6b-ad23-b9778377fe4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d01ec-predictor-8675b95b7b namespace: kserve-ci-e2e-test resourceVersion: "22196" uid: 609ce9a7-47e0-460e-b359-eda840d41edd kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'Created pod: error-404-isvc-d01ec-predictor-8675b95b7b-r5bd2' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor-8675b95b7b.18c6758999b86a2a namespace: kserve-ci-e2e-test resourceVersion: "22206" uid: 00885a61-db96-4980-b227-64757676db44 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22193" uid: c3ba9bd0-3b44-4957-8cf9-5a7ac46e6517 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Scaled up replica set error-404-isvc-d01ec-predictor-8675b95b7b from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec-predictor.18c67589983e8207 namespace: kserve-ci-e2e-test resourceVersion: "22199" uid: c686c6bd-6e3a-4d6f-bccd-c9c6a9d555c3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:59:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: d3926f11-1ab2-47f6-90bb-43dd8a0e1d43 kind: Event lastTimestamp: "2026-07-28T13:00:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:11Z" name: error-404-isvc-d01ec-predictor.18c6758d169f495c namespace: kserve-ci-e2e-test resourceVersion: "22931" uid: 877eef57-1416-426b-bd0b-92b8293d35f9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:59:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22194" uid: d3926f11-1ab2-47f6-90bb-43dd8a0e1d43 kind: Event lastTimestamp: "2026-07-28T13:00:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:11Z" name: error-404-isvc-d01ec-predictor.18c6758d171b61af namespace: kserve-ci-e2e-test resourceVersion: "22932" uid: 26c33c53-706c-4642-8209-46f11810c87d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: d3926f11-1ab2-47f6-90bb-43dd8a0e1d43 kind: Event lastTimestamp: "2026-07-28T12:59:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:26Z" name: error-404-isvc-d01ec-predictor.18c6759095f66056 namespace: kserve-ci-e2e-test resourceVersion: "22541" uid: 7bd41f9e-966a-42e8-86c0-55ba4934e103 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: d3926f11-1ab2-47f6-90bb-43dd8a0e1d43 kind: Event lastTimestamp: "2026-07-28T12:59:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:26Z" name: error-404-isvc-d01ec-predictor.18c675909651c3a9 namespace: kserve-ci-e2e-test resourceVersion: "22543" uid: 477960bd-758e-4537-86ba-59d9de180485 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: ae979099-25e9-4317-8a81-f4a1bd8f9536 kind: Event lastTimestamp: "2026-07-28T12:59:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:59:44Z" name: error-404-isvc-d01ec.18c67589983974ff namespace: kserve-ci-e2e-test resourceVersion: "22696" uid: 565833ad-8c70-4680-86c5-b885cb8239a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: ae979099-25e9-4317-8a81-f4a1bd8f9536 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-d01ec": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d01ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec.18c67589989fe513 namespace: kserve-ci-e2e-test resourceVersion: "22201" uid: dcabbe4f-987d-4948-9165-800a74716491 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: ae979099-25e9-4317-8a81-f4a1bd8f9536 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d01ec": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-d01ec.18c6758998a0435f namespace: kserve-ci-e2e-test resourceVersion: "22202" uid: 4c326a82-1cf4-4b92-86e9-0e61546298be reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: ae979099-25e9-4317-8a81-f4a1bd8f9536 kind: Event lastTimestamp: "2026-07-28T12:59:44Z" message: InferenceService [error-404-isvc-d01ec] is Ready metadata: creationTimestamp: "2026-07-28T12:59:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:59:44Z" name: error-404-isvc-d01ec.18c67594eb67310b namespace: kserve-ci-e2e-test resourceVersion: "22695" uid: 65519480-5b65-4910-9d1e-74ab2012ba04 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:58:06.362220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e02335dce namespace: kserve-ci-e2e-test resourceVersion: "21584" uid: 71bd101a-1e15-43bb-b9a6-4e016aa8692b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-da172-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e1030b11f namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: c6ed2f40-8b50-4536-bd44-02bd2386784e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21590" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Add eth0 [10.132.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e404b1a57 namespace: kserve-ci-e2e-test resourceVersion: "21614" uid: 2d8d5d89-0a57-4eac-a6ff-0639d2b14ea4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e41f33515 namespace: kserve-ci-e2e-test resourceVersion: "21618" uid: 942a83b0-2c10-4a62-8ba8-360be769d090 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e4519666c namespace: kserve-ci-e2e-test resourceVersion: "21621" uid: 27376670-8ee9-46bf-bdeb-32336e35f6b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e4591893f namespace: kserve-ci-e2e-test resourceVersion: "21622" uid: 9576e6b3-ddb2-4532-a994-a49a29c75865 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e459b42bc namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: ce82d34a-76e0-443d-904a-c7221ee293ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e4a320635 namespace: kserve-ci-e2e-test resourceVersion: "21625" uid: 64d4b974-ae25-4b3a-ba51-e81724762977 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e4aa92789 namespace: kserve-ci-e2e-test resourceVersion: "21626" uid: e337f66c-0bb0-4852-83f8-fa174e028c5a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T12:58:44Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:44Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c6757e7cebc161 namespace: kserve-ci-e2e-test resourceVersion: "21961" uid: b0f50eb4-ea88-4584-9edc-9042d3470170 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c675ff36c634e8 namespace: kserve-ci-e2e-test resourceVersion: "26266" uid: 542dac6e-34f1-4651-93bf-a713c9d733ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c675ff36c6ba91 namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: ad023837-a817-4886-a0a1-4cdca04f01bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T13:07:25Z" message: 'Readiness probe failed: Get "https://10.132.0.39:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-28T13:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:25Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c676002d2c6a98 namespace: kserve-ci-e2e-test resourceVersion: "26342" uid: ea903fe9-ad18-4a3c-befa-fd2e742d8506 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2 namespace: kserve-ci-e2e-test resourceVersion: "21582" uid: 4926a647-1726-415e-8835-5f65004ff82e kind: Event lastTimestamp: "2026-07-28T13:07:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.39:8080: i/o timeout' metadata: creationTimestamp: "2026-07-28T13:07:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:25Z" name: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2.18c676002d7cc127 namespace: kserve-ci-e2e-test resourceVersion: "26343" uid: a1d2681b-5f40-4ba2-81c8-dd9a3134fcd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-da172-predictor-f8b85c5dd namespace: kserve-ci-e2e-test resourceVersion: "21572" uid: 38e69adf-fa24-4e8f-b0b1-4e67cb1d87a1 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'Created pod: error-404-isvc-da172-predictor-f8b85c5dd-tp4l2' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172-predictor-f8b85c5dd.18c6757e019f3088 namespace: kserve-ci-e2e-test resourceVersion: "21581" uid: 915cc35b-ec10-4525-a284-ad635f8344b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: e33f47af-dbc1-42d7-ba2b-a95afde81f90 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Scaled up replica set error-404-isvc-da172-predictor-f8b85c5dd from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172-predictor.18c6757dffdf9ef5 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: 8d778083-343e-4a76-b06c-c4b529eda5e5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 95c8e58c-7e49-4f03-9a43-1990bb014e0f kind: Event lastTimestamp: "2026-07-28T12:58:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:51Z" name: error-404-isvc-da172-predictor.18c675817e52c2dd namespace: kserve-ci-e2e-test resourceVersion: "22057" uid: e242858e-7b05-48b8-a138-a8626a3cae2f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21571" uid: 95c8e58c-7e49-4f03-9a43-1990bb014e0f kind: Event lastTimestamp: "2026-07-28T12:58:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:51Z" name: error-404-isvc-da172-predictor.18c675817ebe53b5 namespace: kserve-ci-e2e-test resourceVersion: "22058" uid: 623f890a-183f-4d41-8491-647fbaa75e32 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 95c8e58c-7e49-4f03-9a43-1990bb014e0f kind: Event lastTimestamp: "2026-07-28T12:59:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:21Z" name: error-404-isvc-da172-predictor.18c67584fd6be02a namespace: kserve-ci-e2e-test resourceVersion: "22502" uid: b5e9230a-d40a-4aa0-b0cd-8f4eda2fc1d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21763" uid: 95c8e58c-7e49-4f03-9a43-1990bb014e0f kind: Event lastTimestamp: "2026-07-28T12:59:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:21Z" name: error-404-isvc-da172-predictor.18c67584fdd4ac7b namespace: kserve-ci-e2e-test resourceVersion: "22503" uid: a86ac1f9-87b9-4ae3-bb84-6514827757ee reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 714b26bc-dcfa-404a-8fba-0fd4bce4eb2d kind: Event lastTimestamp: "2026-07-28T12:58:54Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:54Z" name: error-404-isvc-da172.18c6757dffd87d98 namespace: kserve-ci-e2e-test resourceVersion: "22101" uid: 07d8afd1-92a2-43fe-a89a-af4f74ab0f3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 714b26bc-dcfa-404a-8fba-0fd4bce4eb2d kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-da172": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da172": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172.18c6757e007a7ede namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: 70b3b1e8-70df-4ce3-9f6b-e5c9abd1cdea reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 714b26bc-dcfa-404a-8fba-0fd4bce4eb2d kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-da172": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:06Z" name: error-404-isvc-da172.18c6757e007b1a73 namespace: kserve-ci-e2e-test resourceVersion: "21578" uid: b9569e9b-e2f1-460c-b503-5a31b1497ec9 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: 714b26bc-dcfa-404a-8fba-0fd4bce4eb2d kind: Event lastTimestamp: "2026-07-28T12:58:54Z" message: InferenceService [error-404-isvc-da172] is Ready metadata: creationTimestamp: "2026-07-28T12:58:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:54Z" name: error-404-isvc-da172.18c6758937e0717b namespace: kserve-ci-e2e-test resourceVersion: "22100" uid: 930488e2-2d7a-4da4-9008-395f869083b0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:08:21.018334Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26873" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d1e8f5d25 namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: 245f89e8-2d03-4db9-945c-c567b38fedbc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26883" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Add eth0 [10.132.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d38adfb0c namespace: kserve-ci-e2e-test resourceVersion: "26933" uid: abcc0952-f9ff-49bc-84a2-742fb6982954 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3a1b78f3 namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: 78aa55b5-cca7-4409-a9d4-10994febdf9b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3c990672 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 78b0b113-8d10-43d7-aa45-902df49ebdfe reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3d0c59db namespace: kserve-ci-e2e-test resourceVersion: "26941" uid: c9ba848f-ecc6-4184-abf4-f12e360c2476 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3d14d568 namespace: kserve-ci-e2e-test resourceVersion: "26942" uid: 93463d85-231d-4de0-989f-4efc39d7fc7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3f81e929 namespace: kserve-ci-e2e-test resourceVersion: "26943" uid: 7c9b8086-d7f0-47a5-8291-f1ab970155f9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d3ff8a036 namespace: kserve-ci-e2e-test resourceVersion: "26944" uid: 8e227f5c-cf94-4768-805e-b6bf50ac1702 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:08:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:08:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:59Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c6760d9e9a2d02 namespace: kserve-ci-e2e-test resourceVersion: "27422" uid: 58739704-273a-4761-a980-7fb95cef0901 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:25:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:40Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c676ff203d1d82 namespace: kserve-ci-e2e-test resourceVersion: "34531" uid: e5e03e2c-e44e-4649-8383-48754f14fbff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85 namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 507c16a1-485a-40ae-a166-4a0a8802217f kind: Event lastTimestamp: "2026-07-28T13:25:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:40Z" name: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85.18c676ff203dd282 namespace: kserve-ci-e2e-test resourceVersion: "34532" uid: 494a3a29-6330-4597-92b7-bb9ece7059e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-dde97-predictor-db9fd4f9b namespace: kserve-ci-e2e-test resourceVersion: "26865" uid: 3f106d60-450e-4061-b8ed-1393dc707c5a kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created pod: error-404-isvc-dde97-predictor-db9fd4f9b-7kx85' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: error-404-isvc-dde97-predictor-db9fd4f9b.18c6760d1e218504 namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 95254057-be0c-431b-ab13-e8f5ee3c8233 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26862" uid: 0affc246-5f52-46e8-8250-8687b3adc43f kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Scaled up replica set error-404-isvc-dde97-predictor-db9fd4f9b from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:20Z" name: error-404-isvc-dde97-predictor.18c6760d1c989bda namespace: kserve-ci-e2e-test resourceVersion: "26866" uid: d3f30d9f-f2bb-4fa0-8f04-657b74913628 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: c62cae56-0040-40a4-9ea0-2a79724362ca kind: Event lastTimestamp: "2026-07-28T13:08:51Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:51Z" name: error-404-isvc-dde97-predictor.18c676109b19a048 namespace: kserve-ci-e2e-test resourceVersion: "27365" uid: 1a75103c-8a35-4770-bb66-1e7fe24cf7c6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26863" uid: c62cae56-0040-40a4-9ea0-2a79724362ca kind: Event lastTimestamp: "2026-07-28T13:08:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:51Z" name: error-404-isvc-dde97-predictor.18c676109b7d5e14 namespace: kserve-ci-e2e-test resourceVersion: "27366" uid: 14c8a991-53ad-46d6-8915-9fed39bda76a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: c62cae56-0040-40a4-9ea0-2a79724362ca kind: Event lastTimestamp: "2026-07-28T13:09:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:36Z" name: error-404-isvc-dde97-predictor.18c6761798b6db22 namespace: kserve-ci-e2e-test resourceVersion: "27821" uid: aa6c3b1c-030b-47e2-868b-80e78b802883 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27234" uid: c62cae56-0040-40a4-9ea0-2a79724362ca kind: Event lastTimestamp: "2026-07-28T13:09:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:36Z" name: error-404-isvc-dde97-predictor.18c6761799433bed namespace: kserve-ci-e2e-test resourceVersion: "27822" uid: 65520ff0-184c-4ec9-b22e-d7412dd5ae34 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 3a6c0cfe-2549-4785-b72c-1c5009384eb1 kind: Event lastTimestamp: "2026-07-28T13:09:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:09Z" name: error-404-isvc-dde97.18c6760d1c96d9f5 namespace: kserve-ci-e2e-test resourceVersion: "27527" uid: 759f001f-eb1f-413e-9804-0381d1dfaa9f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 3a6c0cfe-2549-4785-b72c-1c5009384eb1 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-dde97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dde97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:20Z" name: error-404-isvc-dde97.18c6760d1d0833c6 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 9d22f20d-15c5-424c-8250-9fd8c9408fa2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26851" uid: 3a6c0cfe-2549-4785-b72c-1c5009384eb1 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-dde97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:20Z" name: error-404-isvc-dde97.18c6760d1d08abd1 namespace: kserve-ci-e2e-test resourceVersion: "26871" uid: 104c66b4-be79-4911-b3fa-4e04dd422309 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27524" uid: 3a6c0cfe-2549-4785-b72c-1c5009384eb1 kind: Event lastTimestamp: "2026-07-28T13:09:09Z" message: InferenceService [error-404-isvc-dde97] is Ready metadata: creationTimestamp: "2026-07-28T13:09:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:09Z" name: error-404-isvc-dde97.18c67618593335a9 namespace: kserve-ci-e2e-test resourceVersion: "27525" uid: 14f5585a-dbc3-475d-8aa7-9f318023faa2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:48:20.981927Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16850" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f7fa9-predictor-7f68c498d-b2czf to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:48:20Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5b6d0e80f namespace: kserve-ci-e2e-test resourceVersion: "16854" uid: 82d87c33-4cc3-492c-aa5b-83de8925ddec reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:21Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f7fa9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:21Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5c3361861 namespace: kserve-ci-e2e-test resourceVersion: "16866" uid: 5548eed6-22a7-4e3f-9071-ace8448b5fe9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16860" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5f496ecea namespace: kserve-ci-e2e-test resourceVersion: "16881" uid: 79abc0c0-e1c4-4cf3-b5ae-1eaae3084fdc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5f6a0db87 namespace: kserve-ci-e2e-test resourceVersion: "16883" uid: a06a319e-4f0d-462b-aa56-28eb7978f28b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5fa816340 namespace: kserve-ci-e2e-test resourceVersion: "16886" uid: d329890e-990f-46cf-a5b7-cfba674c2bf9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5fb1eb93f namespace: kserve-ci-e2e-test resourceVersion: "16887" uid: 45061319-d954-4b41-8ac8-1ce404889bf3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5fb28c789 namespace: kserve-ci-e2e-test resourceVersion: "16888" uid: 13f00d9a-b37a-4baa-9066-13bd2cacef23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5fddd077c namespace: kserve-ci-e2e-test resourceVersion: "16889" uid: 62536409-e888-4cdf-bd8d-d2aca63a44eb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f5fe56ef11 namespace: kserve-ci-e2e-test resourceVersion: "16890" uid: 31ac5a0e-ff04-4279-a384-8e30dd59e642 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:48:59Z" message: 'Readiness probe failed: dial tcp 10.132.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:59Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c674f65f73e4e9 namespace: kserve-ci-e2e-test resourceVersion: "17281" uid: a18b113c-0ec6-4e63-a2d4-0b19e87ae84b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c6757935d3803c namespace: kserve-ci-e2e-test resourceVersion: "21198" uid: d2be7224-3e04-467a-85b0-d12f4b8736cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf namespace: kserve-ci-e2e-test resourceVersion: "16852" uid: 40f65ed7-cd19-4d30-bdb0-a94bb80d36d0 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf.18c6757935d427db namespace: kserve-ci-e2e-test resourceVersion: "21201" uid: 68eb26c9-699c-441d-8c99-bb962deff75e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f7fa9-predictor-7f68c498d namespace: kserve-ci-e2e-test resourceVersion: "16842" uid: 868697f4-a24c-4c70-95bf-0ba478008654 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: 'Created pod: error-404-isvc-f7fa9-predictor-7f68c498d-b2czf' metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:20Z" name: error-404-isvc-f7fa9-predictor-7f68c498d.18c674f5b65b8e4f namespace: kserve-ci-e2e-test resourceVersion: "16851" uid: 3d04d84d-0f77-4a52-83b3-c2243c9e2a81 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 19f184f7-4ade-4467-9414-383bb5e50a92 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Scaled up replica set error-404-isvc-f7fa9-predictor-7f68c498d from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:20Z" name: error-404-isvc-f7fa9-predictor.18c674f5b4bda0f9 namespace: kserve-ci-e2e-test resourceVersion: "16844" uid: 3b22e26e-4dce-4d9b-a94d-de12fa97149e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: a005c4af-eb8a-4ef2-a6d4-1ccbbacaa32d kind: Event lastTimestamp: "2026-07-28T12:49:05Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:05Z" name: error-404-isvc-f7fa9-predictor.18c674f93332a1a1 namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: e3007761-a7c0-489d-92fd-40461f61772e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: a005c4af-eb8a-4ef2-a6d4-1ccbbacaa32d kind: Event lastTimestamp: "2026-07-28T12:49:05Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:05Z" name: error-404-isvc-f7fa9-predictor.18c674f933804342 namespace: kserve-ci-e2e-test resourceVersion: "17328" uid: dbac6197-b2b7-46cf-b73c-ef31e13bdfff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:48:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: a005c4af-eb8a-4ef2-a6d4-1ccbbacaa32d kind: Event lastTimestamp: "2026-07-28T12:49:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:36Z" name: error-404-isvc-f7fa9-predictor.18c674fcb2267bba namespace: kserve-ci-e2e-test resourceVersion: "17596" uid: dfad3245-f68d-4c01-8343-d66e6bcaadc4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:48:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17050" uid: a005c4af-eb8a-4ef2-a6d4-1ccbbacaa32d kind: Event lastTimestamp: "2026-07-28T12:49:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:48:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:36Z" name: error-404-isvc-f7fa9-predictor.18c674fcb2699b17 namespace: kserve-ci-e2e-test resourceVersion: "17597" uid: c90c2553-7ed4-440c-b0aa-dd39a3992534 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: c4fc3e0b-f4cc-49f1-b238-d9bc4e53f10f kind: Event lastTimestamp: "2026-07-28T12:49:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:49:09Z" name: error-404-isvc-f7fa9.18c674f5b4bff9e7 namespace: kserve-ci-e2e-test resourceVersion: "17364" uid: ede2b4b4-e912-4c39-8c8a-d9ea88469ff5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: c4fc3e0b-f4cc-49f1-b238-d9bc4e53f10f kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: 'Failed to update status for InferenceService "error-404-isvc-f7fa9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7fa9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:20Z" name: error-404-isvc-f7fa9.18c674f5b51f0a42 namespace: kserve-ci-e2e-test resourceVersion: "16847" uid: 3759138b-3b16-4a88-b6aa-1edd7c2bc86e reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: c4fc3e0b-f4cc-49f1-b238-d9bc4e53f10f kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f7fa9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:20Z" name: error-404-isvc-f7fa9.18c674f5b51f4967 namespace: kserve-ci-e2e-test resourceVersion: "16848" uid: 67345149-7d84-43b5-9a56-4d906521537b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17362" uid: c4fc3e0b-f4cc-49f1-b238-d9bc4e53f10f kind: Event lastTimestamp: "2026-07-28T12:49:09Z" message: InferenceService [error-404-isvc-f7fa9] is Ready metadata: creationTimestamp: "2026-07-28T12:49:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:49:09Z" name: error-404-isvc-f7fa9.18c675011a115147 namespace: kserve-ci-e2e-test resourceVersion: "17363" uid: 4f042f8a-5c98-4478-8515-e93b5feec132 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:57:45.851936Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21227" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fc1de-predictor-6bc584d948-4fgbk to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c675793bb14966 namespace: kserve-ci-e2e-test resourceVersion: "21231" uid: b247b9e5-46fc-4591-a9b0-c3924855ffd0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fc1de-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c6757945f41a19 namespace: kserve-ci-e2e-test resourceVersion: "21252" uid: 31edeffe-102e-43c7-9a2e-a5704b626a66 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21237" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: Add eth0 [10.132.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579aece7eb0 namespace: kserve-ci-e2e-test resourceVersion: "21279" uid: 1d9e1140-5d23-442a-8e23-a818dba31aaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b028019e namespace: kserve-ci-e2e-test resourceVersion: "21281" uid: 7a6c7898-3f54-486a-a501-5090736dfea4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b300506e namespace: kserve-ci-e2e-test resourceVersion: "21282" uid: 63c533a3-dafd-4690-b748-3c3dd7860378 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b373b832 namespace: kserve-ci-e2e-test resourceVersion: "21283" uid: d9f6abf3-03a9-4818-b995-849e4b1635a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b37d30d2 namespace: kserve-ci-e2e-test resourceVersion: "21284" uid: 32a8fcf9-982d-4c99-b41a-7dbcc98e993d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b6f47f70 namespace: kserve-ci-e2e-test resourceVersion: "21286" uid: 7409420b-75ac-4fe7-bacf-1cb2f1ac79d8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:57:47Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:47Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c67579b768706f namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: ee5c5c7c-2c62-41dd-8e83-6a2e25347a95 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:57:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:58:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:57:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:25Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c6757a0a9a329a namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 5b63f171-4bc8-4156-bb5c-4376e35a813c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c6758993f388f3 namespace: kserve-ci-e2e-test resourceVersion: "22172" uid: c6beedbb-1f87-405d-9ebc-dde70d064d3f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk namespace: kserve-ci-e2e-test resourceVersion: "21229" uid: 12056daa-b8ce-4537-b4dd-626a61607ecb kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk.18c6758993f42175 namespace: kserve-ci-e2e-test resourceVersion: "22175" uid: 5a5676f9-9205-41ff-b968-1384510fcf1f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fc1de-predictor-6bc584d948 namespace: kserve-ci-e2e-test resourceVersion: "21220" uid: 89e7bded-5b93-449f-b15d-30fb394e6260 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'Created pod: error-404-isvc-fc1de-predictor-6bc584d948-4fgbk' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-fc1de-predictor-6bc584d948.18c675793b1db16c namespace: kserve-ci-e2e-test resourceVersion: "21228" uid: d1c746ac-c8d9-43de-b1df-19a121debfe2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 77f51bc7-3cea-477b-8220-d1d2a43d843e kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Scaled up replica set error-404-isvc-fc1de-predictor-6bc584d948 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-fc1de-predictor.18c67579397e40ea namespace: kserve-ci-e2e-test resourceVersion: "21222" uid: 4796e07d-bf98-45e9-b582-7336df810e3f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: e6bbc25e-c006-4c8e-b5fe-8a31bf48d4f3 kind: Event lastTimestamp: "2026-07-28T12:58:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:00Z" name: error-404-isvc-fc1de-predictor.18c6757cb7ed56c8 namespace: kserve-ci-e2e-test resourceVersion: "21441" uid: d833e3bd-5490-4a6a-b492-6f3583470247 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: e6bbc25e-c006-4c8e-b5fe-8a31bf48d4f3 kind: Event lastTimestamp: "2026-07-28T12:58:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:00Z" name: error-404-isvc-fc1de-predictor.18c6757cb850f0f5 namespace: kserve-ci-e2e-test resourceVersion: "21443" uid: 2636335a-5ca1-48ee-a8cf-ad8b8e3ee51b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: e6bbc25e-c006-4c8e-b5fe-8a31bf48d4f3 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: error-404-isvc-fc1de-predictor.18c6758036ed7384 namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: ff36b9c3-d8ba-4ba5-bd51-bf28f4283496 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:58:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21442" uid: e6bbc25e-c006-4c8e-b5fe-8a31bf48d4f3 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: error-404-isvc-fc1de-predictor.18c675803753831e namespace: kserve-ci-e2e-test resourceVersion: "22000" uid: bd8d05a9-805a-4356-a148-8c43b8250433 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 1a8e1dad-522d-4e60-8c97-dfe0f1ee8bf6 kind: Event lastTimestamp: "2026-07-28T12:58:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:35Z" name: error-404-isvc-fc1de.18c67579397b0ceb namespace: kserve-ci-e2e-test resourceVersion: "21886" uid: cc9dde2c-27b1-4fab-bd18-deebd68e3f7e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 1a8e1dad-522d-4e60-8c97-dfe0f1ee8bf6 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'Failed to update status for InferenceService "error-404-isvc-fc1de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-fc1de.18c6757939fdc7a6 namespace: kserve-ci-e2e-test resourceVersion: "21224" uid: 905975ac-25b1-4021-96c6-0f667a772d6c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21200" uid: 1a8e1dad-522d-4e60-8c97-dfe0f1ee8bf6 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: error-404-isvc-fc1de.18c6757939fe15e6 namespace: kserve-ci-e2e-test resourceVersion: "21225" uid: d7b086bb-b6c0-4133-906a-03ccc62481cc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21884" uid: 1a8e1dad-522d-4e60-8c97-dfe0f1ee8bf6 kind: Event lastTimestamp: "2026-07-28T12:58:35Z" message: InferenceService [error-404-isvc-fc1de] is Ready metadata: creationTimestamp: "2026-07-28T12:58:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:35Z" name: error-404-isvc-fc1de.18c67584c570014d namespace: kserve-ci-e2e-test resourceVersion: "21885" uid: e939b76c-b923-4f80-8350-903748328679 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:47:41.227224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16271" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ec75403c55 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 48b210fe-3368-4bed-9c01-9368561a9fa6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-fc405-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ec8240a156 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 0fbb4e13-7518-4659-a68c-adfeae552c21 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16288" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecb2b3dbc5 namespace: kserve-ci-e2e-test resourceVersion: "16326" uid: 7caedc2f-6aac-4cb5-a6cd-b5da22d381fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecb412ba17 namespace: kserve-ci-e2e-test resourceVersion: "16328" uid: b215f083-6697-445d-9c79-3d6f0c113d18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecb6d4d8da namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 5c319f43-f491-4eea-86e9-b95e178e9cdf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecb75c27b2 namespace: kserve-ci-e2e-test resourceVersion: "16332" uid: 67154478-00fc-47a4-8385-31d78febdc78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecb7665f62 namespace: kserve-ci-e2e-test resourceVersion: "16333" uid: bb55fab4-4116-4e57-b590-64742fa70eb3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecba6baa1c namespace: kserve-ci-e2e-test resourceVersion: "16334" uid: 9534c006-d213-4faf-8110-c59a40bdae3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecbad0e3ce namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: a84ec7ce-62ff-4f5d-8a27-786895658b6c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:56:58Z" message: 'Readiness probe failed: dial tcp 10.132.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:58Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c674ecc82c9861 namespace: kserve-ci-e2e-test resourceVersion: "20755" uid: 2168dd81-338c-49a0-928c-4d9acf6d10b4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c6756da2ce9e97 namespace: kserve-ci-e2e-test resourceVersion: "20689" uid: c51f6f4a-b7df-4206-8a7d-2ed0e9e80c07 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:56Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c6756da2cf475f namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 3228bafe-211d-49c4-a3ce-b59caef308d7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: 377a0412-19a2-43c5-a249-a7edecac9337 kind: Event lastTimestamp: "2026-07-28T12:56:58Z" message: 'Readiness probe failed: Get "https://10.132.0.31:8643/healthz": dial tcp 10.132.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:56:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:58Z" name: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg.18c6756e3c8717bd namespace: kserve-ci-e2e-test resourceVersion: "20754" uid: 66281187-5bda-4070-8a79-3fd2c36b592e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-fc405-predictor-86dddcdd4b namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 498afd4c-808c-404d-9ecb-cd7d9995638f kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'Created pod: error-404-isvc-fc405-predictor-86dddcdd4b-hh8lg' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405-predictor-86dddcdd4b.18c674ec748cef0b namespace: kserve-ci-e2e-test resourceVersion: "16272" uid: 3e5274a5-12bd-4fcd-bb1e-8d72589ea588 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16258" uid: 66573be6-3954-44ba-b248-8f55066a6ac2 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Scaled up replica set error-404-isvc-fc405-predictor-86dddcdd4b from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405-predictor.18c674ec71f7cbf6 namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 4d81058e-dd47-4d90-9f34-5e1a5234aca2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:47:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 084a31c6-ff99-4abd-a671-3ec6809f44b2 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:41Z" name: error-404-isvc-fc405-predictor.18c674eff0ad99f6 namespace: kserve-ci-e2e-test resourceVersion: "17118" uid: 85fecb55-5069-4bee-a3f9-fb0d1e6f9ea6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:47:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16260" uid: 084a31c6-ff99-4abd-a671-3ec6809f44b2 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:41Z" name: error-404-isvc-fc405-predictor.18c674eff125b8c4 namespace: kserve-ci-e2e-test resourceVersion: "17119" uid: a00808b4-6fa5-40f9-9c92-4b937c7667c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 084a31c6-ff99-4abd-a671-3ec6809f44b2 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:11Z" name: error-404-isvc-fc405-predictor.18c674f36fdd6739 namespace: kserve-ci-e2e-test resourceVersion: "16640" uid: c6549bd6-41b9-4d50-9b25-3dbcc62fdc99 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 084a31c6-ff99-4abd-a671-3ec6809f44b2 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:11Z" name: error-404-isvc-fc405-predictor.18c674f37048934a namespace: kserve-ci-e2e-test resourceVersion: "16642" uid: 90232817-a6db-4955-8886-0fcbcc809047 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 73b66ea1-e13c-4182-8d03-adc6d1c391ce kind: Event lastTimestamp: "2026-07-28T12:48:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:28Z" name: error-404-isvc-fc405.18c674ec720424fb namespace: kserve-ci-e2e-test resourceVersion: "16982" uid: a25803db-1012-4a56-a1dd-ecdb43ecbd85 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 73b66ea1-e13c-4182-8d03-adc6d1c391ce kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-fc405": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc405": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405.18c674ec72cb0b1d namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: b02c8343-d0e6-4eee-8ff8-13b4369904c7 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16229" uid: 73b66ea1-e13c-4182-8d03-adc6d1c391ce kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-fc405": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:41Z" name: error-404-isvc-fc405.18c674ec72cb7b13 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 7a3f57f2-f7c4-473e-90ba-4da72f9c7cb6 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16980" uid: 73b66ea1-e13c-4182-8d03-adc6d1c391ce kind: Event lastTimestamp: "2026-07-28T12:48:28Z" message: InferenceService [error-404-isvc-fc405] is Ready metadata: creationTimestamp: "2026-07-28T12:48:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:28Z" name: error-404-isvc-fc405.18c674f78440935b namespace: kserve-ci-e2e-test resourceVersion: "16981" uid: cbf8bd98-566a-41e1-9556-c404eb4a09ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:46:00.766646Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-645b44c55c-znpfs to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:46:00Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d51155800f namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: c250ea66-22ce-48b3-ba25-d86a367e5a75 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:00Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d51e30bce8 namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: 872e2dfd-6ace-47f9-8782-6bfd61f17816 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15152" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Add eth0 [10.134.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d54fcf2c7a namespace: kserve-ci-e2e-test resourceVersion: "15251" uid: 677e7f89-07c6-41b4-94fb-c593c5f3670f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:45d91dc105b49144984afdbe9de3160da1c989d254fd58cacbd7a1fbe0d83b56" already present on machine metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d551753859 namespace: kserve-ci-e2e-test resourceVersion: "15253" uid: 71169bcd-a453-4f9b-9473-933607a9410e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d55725a592 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 306f4f04-6f59-4bf2-8185-38ae00475f3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d55851259e namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 980ade32-683c-410e-a9be-5975a5a4981a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1146" metadata: creationTimestamp: "2026-07-28T12:46:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:06Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d66aa5f77e namespace: kserve-ci-e2e-test resourceVersion: "15318" uid: 0c2fb0bb-7e8a-413f-ac33-03abf7568b48 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1146" in 11.762s (11.762s including waiting). Image size: 1555707315 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d927bfbabe namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d34ddf33-9563-4c45-aa4d-e2cbbab0b6d1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:19Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d95f29842a namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: dee6721e-29b7-4273-bd14-d9b776b7f620 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:19Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d95ff61e60 namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: c4377cd7-2e02-479d-ab81-d45816160ae9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T12:46:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:19Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674d960048d4d namespace: kserve-ci-e2e-test resourceVersion: "15461" uid: 4ee19afa-ee69-4205-8b57-f488cff42ff0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.021s (4.021s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674da4fb4e1bc namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: 63e9bc7e-095e-429c-9178-2673b79f8636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674da577efd03 namespace: kserve-ci-e2e-test resourceVersion: "15497" uid: e63d6d95-075a-4e99-bd48-ee812853cc5d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674da5941a871 namespace: kserve-ci-e2e-test resourceVersion: "15499" uid: 02f904c2-d30f-4df0-ac58-1fdaca2b0408 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: 'Readiness probe failed: dial tcp 10.134.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674da68369540 namespace: kserve-ci-e2e-test resourceVersion: "16893" uid: fa9bba3f-20ba-4fc0-90af-7049b272992b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674f5b0f61d50 namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: b9e051eb-f8cb-4a50-bf68-cb6734257ad1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674f5b0f6cd8a namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: efdffa06-23f9-40de-80ef-e6bb8cc3803a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs namespace: kserve-ci-e2e-test resourceVersion: "15134" uid: 2f27dff8-0f51-4b0e-9af7-7da36d17fe53 kind: Event lastTimestamp: "2026-07-28T12:48:24Z" message: 'Readiness probe failed: Get "https://10.134.0.28:8643/healthz": dial tcp 10.134.0.28:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:24Z" name: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs.18c674f694dabb39 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: c55efb5d-06ad-4de3-a93c-0176b4f74ca0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-645b44c55c namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: e662dcc6-5d40-4d24-856f-5ddaa2343eba kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-645b44c55c-znpfs' metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: isvc-sklearn-graph-1-predictor-645b44c55c.18c674d510e69ec2 namespace: kserve-ci-e2e-test resourceVersion: "15133" uid: 31ba42b1-97ed-4c13-94df-24cf2799c6c9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15123" uid: fd4b020f-62dd-463f-bb95-33109f8fde29 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-645b44c55c from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: isvc-sklearn-graph-1-predictor.18c674d50dbe71cc namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: 1883e52e-bbb3-4de9-93ec-a9c39c987928 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 2f184ff1-7517-4702-b07a-aa17421262c2 kind: Event lastTimestamp: "2026-07-28T12:47:00Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:00Z" name: isvc-sklearn-graph-1-predictor.18c674d88c4aacdc namespace: kserve-ci-e2e-test resourceVersion: "15815" uid: b52109a7-6152-4887-a358-e8248fca61fb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 2f184ff1-7517-4702-b07a-aa17421262c2 kind: Event lastTimestamp: "2026-07-28T12:47:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:00Z" name: isvc-sklearn-graph-1-predictor.18c674d88cac6758 namespace: kserve-ci-e2e-test resourceVersion: "15816" uid: 626cf306-316e-4bb7-8f90-c2bf04096bf0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 2f184ff1-7517-4702-b07a-aa17421262c2 kind: Event lastTimestamp: "2026-07-28T12:48:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:00Z" name: isvc-sklearn-graph-1-predictor.18c674e687c2463c namespace: kserve-ci-e2e-test resourceVersion: "16531" uid: 09226a11-b224-4990-8777-1588d9c60ed7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:47:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15411" uid: 2f184ff1-7517-4702-b07a-aa17421262c2 kind: Event lastTimestamp: "2026-07-28T12:48:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:00Z" name: isvc-sklearn-graph-1-predictor.18c674e68846c15d namespace: kserve-ci-e2e-test resourceVersion: "16532" uid: 14cc6b80-0e9f-4c94-93ed-df7ebf31f808 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15111" uid: 9aa7b329-5d3a-4a42-a10c-9eb5119ecafd kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:42Z" name: isvc-sklearn-graph-1.18c674d50db0048a namespace: kserve-ci-e2e-test resourceVersion: "16350" uid: 1d12c19f-b8c5-4ace-8519-6f583458cc52 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "16348" uid: 9aa7b329-5d3a-4a42-a10c-9eb5119ecafd kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:42Z" name: isvc-sklearn-graph-1.18c674ecd094d5a3 namespace: kserve-ci-e2e-test resourceVersion: "16349" uid: 37e63b63-904c-40f1-a8b0-5f9961b54a2b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:46:01.070104Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d5236be22e namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: cf7e924b-9f52-4f72-8a74-31da4f58f05b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d53040a60e namespace: kserve-ci-e2e-test resourceVersion: "15235" uid: 630e47f1-46bf-40d3-bdec-74f993796f73 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15223" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:46:02Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d567a9f463 namespace: kserve-ci-e2e-test resourceVersion: "15264" uid: 48ce7a53-0a0f-448e-9ffd-019d7e09da68 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:45d91dc105b49144984afdbe9de3160da1c989d254fd58cacbd7a1fbe0d83b56" already present on machine metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d56a329939 namespace: kserve-ci-e2e-test resourceVersion: "15267" uid: dc9f0397-a763-44d5-96be-5e7fca98e749 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d56edfc57e namespace: kserve-ci-e2e-test resourceVersion: "15269" uid: 20e7c707-abde-486d-8d7f-fdc8746f66e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d570f48e8f namespace: kserve-ci-e2e-test resourceVersion: "15270" uid: 33a61bd9-55f4-4158-8cb2-b867e01dd915 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:07Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1146" metadata: creationTimestamp: "2026-07-28T12:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:07Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d6a6f59f51 namespace: kserve-ci-e2e-test resourceVersion: "15329" uid: a77e4365-71c7-48ff-9b80-98045e28491f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1146" in 10.745s (10.745s including waiting). Image size: 1555707315 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d9276b5472 namespace: kserve-ci-e2e-test resourceVersion: "15445" uid: 7cee7be7-f07d-4100-b9ca-af5543345e44 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d92cbb30a6 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: 57656b58-7848-4979-87c8-488c84e5d16d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d92d6b30ee namespace: kserve-ci-e2e-test resourceVersion: "15450" uid: 09036ecb-7621-42f9-b1da-66e2a9e59de2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674d92d7cb7d2 namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: 23fbdefc-7cf1-44a7-92a9-def9ea7efe8e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.731s (4.731s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674da4789096c namespace: kserve-ci-e2e-test resourceVersion: "15489" uid: 0c9bc1d7-7163-4032-b6e9-778a91a50e94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674da56a62fc0 namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: f860f828-550a-4c46-a1b7-49fdb0a46ffc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674da57d9a609 namespace: kserve-ci-e2e-test resourceVersion: "15498" uid: a7480e23-f983-456f-ada4-97eafa055edf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T12:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:47:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:40Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674daa44e1698 namespace: kserve-ci-e2e-test resourceVersion: "16189" uid: 1cb28706-6333-4198-bc27-72d6f91320f6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674f5ad4f3500 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: a9b74883-93cf-4dd8-b472-e46f01f0f41f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674f5ad4fb62d namespace: kserve-ci-e2e-test resourceVersion: "16796" uid: 11aa3a38-4c70-484f-8bdc-b88505dfdeca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh namespace: kserve-ci-e2e-test resourceVersion: "15213" uid: 0b1d9b25-5f04-47d9-8368-ed2d7b609be3 kind: Event lastTimestamp: "2026-07-28T12:48:25Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:48:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:25Z" name: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh.18c674f6d0c5f447 namespace: kserve-ci-e2e-test resourceVersion: "16940" uid: a08683e7-00eb-48a8-8dc2-4788b0ea3dc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-544798b8f5 namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: eb5853dc-9a97-4d21-83cb-7575aeccc728 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-544798b8f5-kq6xh' metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-2-predictor-544798b8f5.18c674d522efb3c8 namespace: kserve-ci-e2e-test resourceVersion: "15212" uid: 819cce00-4ee4-41f8-bb94-3fab5741e406 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: b85eabfe-92d9-44c3-adca-06289f007760 kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-544798b8f5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:01Z" name: isvc-sklearn-graph-2-predictor.18c674d5208c972a namespace: kserve-ci-e2e-test resourceVersion: "15206" uid: a7a7fbfe-a124-4659-8b00-21b8e112f3d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: 9ae7186b-a052-454c-a446-52586c21bceb kind: Event lastTimestamp: "2026-07-28T12:46:31Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:31Z" name: isvc-sklearn-graph-2-predictor.18c674d89eff0166 namespace: kserve-ci-e2e-test resourceVersion: "15577" uid: 3bd1c80f-fe41-445e-9385-f36666128ae6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:46:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: 9ae7186b-a052-454c-a446-52586c21bceb kind: Event lastTimestamp: "2026-07-28T12:46:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:31Z" name: isvc-sklearn-graph-2-predictor.18c674d89f481aa9 namespace: kserve-ci-e2e-test resourceVersion: "15578" uid: 15f8cc50-5a48-4a13-8e7e-e0898e685e56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 9ae7186b-a052-454c-a446-52586c21bceb kind: Event lastTimestamp: "2026-07-28T12:48:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:01Z" name: isvc-sklearn-graph-2-predictor.18c674df9cb14c52 namespace: kserve-ci-e2e-test resourceVersion: "16536" uid: f0144fe9-5eff-4732-ad32-49d7bee720a9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:46:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: 9ae7186b-a052-454c-a446-52586c21bceb kind: Event lastTimestamp: "2026-07-28T12:48:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:46:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:01Z" name: isvc-sklearn-graph-2-predictor.18c674df9cfe284b namespace: kserve-ci-e2e-test resourceVersion: "16537" uid: 231899bb-e357-4c53-bd31-83ad0be71640 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15182" uid: 834b0069-20a8-4418-85a2-57c459da6e48 kind: Event lastTimestamp: "2026-07-28T12:47:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:48Z" name: isvc-sklearn-graph-2.18c674d52082b183 namespace: kserve-ci-e2e-test resourceVersion: "16430" uid: ed03c3e6-8a5b-4b04-b5ce-7bf5c19038e2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "16428" uid: 834b0069-20a8-4418-85a2-57c459da6e48 kind: Event lastTimestamp: "2026-07-28T12:47:48Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T12:47:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:48Z" name: isvc-sklearn-graph-2.18c674ee37d2e865 namespace: kserve-ci-e2e-test resourceVersion: "16429" uid: e895d6fe-b6b7-4592-903c-e8cc75f556d3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:46:00.967810Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-7zt6m to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:46:00Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d51d531380 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 77fa4f3b-97f0-4cf6-8066-79d727e1c498 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:01Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:01Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:46:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:01Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d52a3f29e6 namespace: kserve-ci-e2e-test resourceVersion: "15228" uid: 94429770-6a02-4467-bc08-71c3d0cf3d89 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Add eth0 [10.134.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:46:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d56172a891 namespace: kserve-ci-e2e-test resourceVersion: "15260" uid: 63c08405-ebe5-45e7-ab62-feebdf3372ae reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:45d91dc105b49144984afdbe9de3160da1c989d254fd58cacbd7a1fbe0d83b56" already present on machine metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d563f649b4 namespace: kserve-ci-e2e-test resourceVersion: "15262" uid: 41950ca4-8600-4670-b27e-1538b4003d90 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d56941d467 namespace: kserve-ci-e2e-test resourceVersion: "15266" uid: 49506584-8750-422e-b66b-e81187bc061c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T12:46:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:02Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d56b5f1cc1 namespace: kserve-ci-e2e-test resourceVersion: "15268" uid: 4fed95f4-ffe5-4fb9-83ac-f79cb46bee56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:07Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T12:46:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674d6a74868ea namespace: kserve-ci-e2e-test resourceVersion: "15330" uid: 1e8b82c9-a250-4c0a-b8e5-876e3dafe3de reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 29.45s (29.45s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd82accbd3 namespace: kserve-ci-e2e-test resourceVersion: "15620" uid: b4cb6c31-2c92-4115-92c7-66e795a39452 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd86af8e74 namespace: kserve-ci-e2e-test resourceVersion: "15621" uid: d264730b-7687-496a-a184-5b1ca8e0b135 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd872b0cf9 namespace: kserve-ci-e2e-test resourceVersion: "15622" uid: 0dc22244-1a74-4ca1-b9aa-0049a0297f15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd87416b2e namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: 5ba2f355-d51b-4171-a89a-0be3e26e1523 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd8b817370 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: 8dbaec09-8122-40fe-bedd-7bcb36142b7d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:46:37Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:46:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:37Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674dd8c3f777a namespace: kserve-ci-e2e-test resourceVersion: "15627" uid: d23f26df-2b72-4a27-8b9b-0c25845a6bbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T12:46:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:48:24Z" message: 'Readiness probe failed: dial tcp 10.134.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:46:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:24Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674ddebe05c27 namespace: kserve-ci-e2e-test resourceVersion: "16929" uid: e64a089f-c83b-46f8-805a-b0cc017ddaf6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674f5a9fdd5cb namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: a9004d26-f0aa-4617-adda-3883bc493210 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674f5a9fe4e74 namespace: kserve-ci-e2e-test resourceVersion: "16772" uid: bdedf007-8265-4f4c-b733-9153d5622ad6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: 25d351f8-741a-457e-95ba-06e8863eb0ba kind: Event lastTimestamp: "2026-07-28T12:48:24Z" message: 'Readiness probe failed: Get "https://10.134.0.29:8643/healthz": dial tcp 10.134.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:48:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:24Z" name: isvc-xgboost-graph-predictor-555fc69d56-7zt6m.18c674f69a3ee18e namespace: kserve-ci-e2e-test resourceVersion: "16928" uid: 75cc5fff-3064-4938-b454-4bbf5f9de7b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15175" uid: 712f3d15-1dee-4ce6-b915-92b1a6df1bb1 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-7zt6m' metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c674d51cca8f9d namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: b5fb4a44-5d02-4a23-bce7-a280a47e25a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15173" uid: 6fa6b775-8363-48c2-8db6-029119edc881 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: isvc-xgboost-graph-predictor.18c674d51980c3d2 namespace: kserve-ci-e2e-test resourceVersion: "15177" uid: d634e033-a1b9-4935-9b11-5bc93edfcd32 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: a8811b86-2b33-41c1-a14e-d435a4a0a25e kind: Event lastTimestamp: "2026-07-28T12:47:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:15Z" name: isvc-xgboost-graph-predictor.18c674d897f7aba6 namespace: kserve-ci-e2e-test resourceVersion: "15961" uid: d96d81b4-0393-47b2-8345-2cdad00efb20 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15174" uid: a8811b86-2b33-41c1-a14e-d435a4a0a25e kind: Event lastTimestamp: "2026-07-28T12:47:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:15Z" name: isvc-xgboost-graph-predictor.18c674d898615269 namespace: kserve-ci-e2e-test resourceVersion: "15962" uid: c30c5873-d8e6-4446-8ceb-ec33ff66aa52 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T12:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: a8811b86-2b33-41c1-a14e-d435a4a0a25e kind: Event lastTimestamp: "2026-07-28T12:48:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:16Z" name: isvc-xgboost-graph-predictor.18c674e314866543 namespace: kserve-ci-e2e-test resourceVersion: "16687" uid: 6f0f3c4d-a246-4fe6-9604-1ee2d11f3a42 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T12:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: a8811b86-2b33-41c1-a14e-d435a4a0a25e kind: Event lastTimestamp: "2026-07-28T12:48:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:16Z" name: isvc-xgboost-graph-predictor.18c674e314f1f0c3 namespace: kserve-ci-e2e-test resourceVersion: "16688" uid: 8fa7d7fa-351d-4b01-a437-bc0bdf9feca3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: bbfd274a-3db7-4b78-ae7f-603081b0401f kind: Event lastTimestamp: "2026-07-28T12:47:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:44Z" name: isvc-xgboost-graph.18c674d519804584 namespace: kserve-ci-e2e-test resourceVersion: "16379" uid: c0428273-9f48-49e7-9bcb-b11fa3d1e9be reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "16377" uid: bbfd274a-3db7-4b78-ae7f-603081b0401f kind: Event lastTimestamp: "2026-07-28T12:47:44Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T12:47:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:44Z" name: isvc-xgboost-graph.18c674ed4fbf45e7 namespace: kserve-ci-e2e-test resourceVersion: "16378" uid: 09d34e59-928c-406d-9c81-5c2bf61fe7ef reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:48:10.546137Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16615" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-58dff6d8d-9q9x9 to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:48:10Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:48:10Z" name: model-chainer-58dff6d8d-9q9x9.18c674f348cb5f5e namespace: kserve-ci-e2e-test resourceVersion: "16620" uid: dc701988-2636-4ddb-a794-d3e1deda3218 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:10Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:10Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:10Z" name: model-chainer-58dff6d8d-9q9x9.18c674f357f4c8cd namespace: kserve-ci-e2e-test resourceVersion: "16631" uid: 8fa661c8-c36c-4caf-b0a4-32d164da205b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16624" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:48:11Z" name: model-chainer-58dff6d8d-9q9x9.18c674f386eff61b namespace: kserve-ci-e2e-test resourceVersion: "16647" uid: 8744cfa8-3382-4e2e-91b8-adfc380ccc36 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:11Z" name: model-chainer-58dff6d8d-9q9x9.18c674f38882097c namespace: kserve-ci-e2e-test resourceVersion: "16649" uid: d3c3f552-db60-4018-a029-3c76133d4c12 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:11Z" name: model-chainer-58dff6d8d-9q9x9.18c674f38bd105a8 namespace: kserve-ci-e2e-test resourceVersion: "16651" uid: af38587f-fbfb-4a45-b049-8f7e0be44565 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:11Z" name: model-chainer-58dff6d8d-9q9x9.18c674f38c4f6987 namespace: kserve-ci-e2e-test resourceVersion: "16652" uid: 428c179c-4fdf-420b-92d9-fc55b3e52b22 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: model-chainer-58dff6d8d-9q9x9.18c674f5a4a9e741 namespace: kserve-ci-e2e-test resourceVersion: "16752" uid: c2c3d93e-3413-4353-8966-5756e015b212 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-58dff6d8d-9q9x9 namespace: kserve-ci-e2e-test resourceVersion: "16617" uid: 82874998-e936-440e-9bde-8d6035667648 kind: Event lastTimestamp: "2026-07-28T12:48:47Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:47Z" name: model-chainer-58dff6d8d-9q9x9.18c674f620e1f309 namespace: kserve-ci-e2e-test resourceVersion: "17178" uid: 47ab7ac6-dea9-473a-9da9-e9d1deb12fc6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:10Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-58dff6d8d namespace: kserve-ci-e2e-test resourceVersion: "16612" uid: 48cf5da0-9c8f-414c-a315-81acd8006311 kind: Event lastTimestamp: "2026-07-28T12:48:10Z" message: 'Created pod: model-chainer-58dff6d8d-9q9x9' metadata: creationTimestamp: "2026-07-28T12:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:10Z" name: model-chainer-58dff6d8d.18c674f3486131a6 namespace: kserve-ci-e2e-test resourceVersion: "16616" uid: 89c50629-9d0f-4277-883a-19aaf6cf08aa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:10Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16610" uid: d520e988-9d14-4a8b-abb7-ce81f152e008 kind: Event lastTimestamp: "2026-07-28T12:48:10Z" message: Scaled up replica set model-chainer-58dff6d8d from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:10Z" name: model-chainer.18c674f3478a7d53 namespace: kserve-ci-e2e-test resourceVersion: "16613" uid: 19fb7f63-e89d-4651-bcac-015f6a0213bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:10Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16627" uid: 71d7e427-adb2-4409-9054-3ef00f3710d6 kind: Event lastTimestamp: "2026-07-28T12:48:10Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T12:48:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:10Z" name: model-chainer.18c674f34c6d658f namespace: kserve-ci-e2e-test resourceVersion: "16629" uid: 49635cd8-e337-4103-813b-12bbb1326a78 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:00:06.137982Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22867" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-d01ec-59754655f9-2xm8b to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T13:00:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:00:06Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c67599e56470a3 namespace: kserve-ci-e2e-test resourceVersion: "22872" uid: fd18ef65-f81f-481d-aa70-1542abd353f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:06Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:00:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-d01ec-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:00:06Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c67599ef6e89f0 namespace: kserve-ci-e2e-test resourceVersion: "22882" uid: 204be310-9a8d-445c-a45c-966906188dd4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:07Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22876" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:00:07Z" message: Add eth0 [10.132.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:00:07Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6759a22d0861b namespace: kserve-ci-e2e-test resourceVersion: "22893" uid: 6a8e1426-7bda-42c8-a9b5-d18a57b72b74 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d01ec} kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:00:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:00:07Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6759a2425f74a namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: 4dd99604-0cf0-4920-867c-996c7dfac605 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d01ec} kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:00:07Z" message: 'Created container: sequence-graph-d01ec' metadata: creationTimestamp: "2026-07-28T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:00:07Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6759a279d098a namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: b98f82b8-c43a-43f1-abe2-6d85cd3ae17a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d01ec} kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:00:07Z" message: Started container sequence-graph-d01ec metadata: creationTimestamp: "2026-07-28T13:00:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:00:07Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6759a2817b01f namespace: kserve-ci-e2e-test resourceVersion: "22897" uid: 42cf994a-dddc-46aa-9a72-d71b1a725970 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d01ec} kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Stopping container sequence-graph-d01ec metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:20Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6760d0c7d1cd0 namespace: kserve-ci-e2e-test resourceVersion: "26791" uid: a07e595c-fc0e-497c-b623-24c918bc812f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:08:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-d01ec} kind: Pod name: sequence-graph-d01ec-59754655f9-2xm8b namespace: kserve-ci-e2e-test resourceVersion: "22870" uid: c1878b56-1edb-4f07-af9c-b7c807c95532 kind: Event lastTimestamp: "2026-07-28T13:08:48Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T13:08:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:48Z" name: sequence-graph-d01ec-59754655f9-2xm8b.18c6760dcf54234f namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: edd8b669-ed6a-4d43-9bb8-08a30ba136e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-d01ec-59754655f9 namespace: kserve-ci-e2e-test resourceVersion: "22864" uid: 6e543830-b385-40ce-a050-c958957be43b kind: Event lastTimestamp: "2026-07-28T13:00:06Z" message: 'Created pod: sequence-graph-d01ec-59754655f9-2xm8b' metadata: creationTimestamp: "2026-07-28T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:06Z" name: sequence-graph-d01ec-59754655f9.18c67599e502ff99 namespace: kserve-ci-e2e-test resourceVersion: "22869" uid: 7cabd795-08e8-42f1-94a0-a4ff13fc6bd9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22862" uid: 84277587-a827-440b-9d73-ee66bff42372 kind: Event lastTimestamp: "2026-07-28T13:00:06Z" message: Scaled up replica set sequence-graph-d01ec-59754655f9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:06Z" name: sequence-graph-d01ec.18c67599e4165ed7 namespace: kserve-ci-e2e-test resourceVersion: "22865" uid: bd580c74-7dba-4d33-b4fb-da4b4a45c9eb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22878" uid: 7f666a86-f43b-42a0-b6cf-49ff56b24848 kind: Event lastTimestamp: "2026-07-28T13:00:06Z" message: InferenceGraph [sequence-graph-d01ec] is Ready metadata: creationTimestamp: "2026-07-28T13:00:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:00:06Z" name: sequence-graph-d01ec.18c67599eb4ebcfa namespace: kserve-ci-e2e-test resourceVersion: "22879" uid: 8026f04b-9188-4d68-bef6-05a64b2048b3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22863" uid: 33fb2019-1935-4f93-95ed-c9499e21cf98 kind: Event lastTimestamp: "2026-07-28T13:00:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:21Z" name: sequence-graph-d01ec.18c6759d632122e9 namespace: kserve-ci-e2e-test resourceVersion: "23001" uid: 31a0f721-7bf5-42c9-a336-1880b9054c8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22863" uid: 33fb2019-1935-4f93-95ed-c9499e21cf98 kind: Event lastTimestamp: "2026-07-28T13:00:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:00:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:21Z" name: sequence-graph-d01ec.18c6759d6396da65 namespace: kserve-ci-e2e-test resourceVersion: "23003" uid: 1b1c128e-9589-4580-8d4e-f5c61561d8e9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "23000" uid: 33fb2019-1935-4f93-95ed-c9499e21cf98 kind: Event lastTimestamp: "2026-07-28T13:00:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:36Z" name: sequence-graph-d01ec.18c675a0e2f976fc namespace: kserve-ci-e2e-test resourceVersion: "23107" uid: a27e8b4d-15f1-4526-8bec-6c592d035855 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:00:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-d01ec namespace: kserve-ci-e2e-test resourceVersion: "23000" uid: 33fb2019-1935-4f93-95ed-c9499e21cf98 kind: Event lastTimestamp: "2026-07-28T13:00:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:00:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:00:36Z" name: sequence-graph-d01ec.18c675a0e39dde00 namespace: kserve-ci-e2e-test resourceVersion: "23109" uid: 1d0d8840-f846-4e91-9dd6-2494e49f2f6c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:49:30.904382Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17533" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-f7fa9-649ffd79b6-dt6lz to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:49:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:49:30Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c67505fe84e9a7 namespace: kserve-ci-e2e-test resourceVersion: "17538" uid: 50609b9d-2d5e-4f6a-912a-d22904bc06e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:49:31Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-f7fa9-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:49:31Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c6750608aa0290 namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 8f170807-5052-49bc-9783-ca3d1b2b7dda reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:31Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17542" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:49:31Z" message: Add eth0 [10.132.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:49:31Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c675063c585bcc namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 069b2b2a-6916-4ae5-b153-58483b02f8be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7fa9} kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:49:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:49:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:49:31Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c675063dceb375 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 9cccc43d-8e8c-4ecf-b2e8-a2984ba60210 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7fa9} kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:49:32Z" message: 'Created container: sequence-graph-f7fa9' metadata: creationTimestamp: "2026-07-28T12:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:49:32Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c6750641395eee namespace: kserve-ci-e2e-test resourceVersion: "17563" uid: 1d8dbbae-7921-445e-98c1-875493badcec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7fa9} kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:49:32Z" message: Started container sequence-graph-f7fa9 metadata: creationTimestamp: "2026-07-28T12:49:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:49:32Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c6750641bb1dfb namespace: kserve-ci-e2e-test resourceVersion: "17564" uid: 5c8f6bf0-c1da-4dd4-8373-9fa4d54ab20a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7fa9} kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Stopping container sequence-graph-f7fa9 metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:45Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c675792cf56952 namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: 8ea3f2bb-b2a5-4aa3-95ae-8087b0004c87 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:57:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-f7fa9} kind: Pod name: sequence-graph-f7fa9-649ffd79b6-dt6lz namespace: kserve-ci-e2e-test resourceVersion: "17534" uid: 6565bb02-bd4e-4b97-ab06-fbf3ce9b6f5f kind: Event lastTimestamp: "2026-07-28T12:58:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:13Z" name: sequence-graph-f7fa9-649ffd79b6-dt6lz.18c67579c34daf18 namespace: kserve-ci-e2e-test resourceVersion: "21683" uid: 82fb3835-26b7-45f7-90e9-9547d4594311 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-f7fa9-649ffd79b6 namespace: kserve-ci-e2e-test resourceVersion: "17530" uid: 7e9f5f9b-ddc2-40eb-af54-90c0caa6a3bd kind: Event lastTimestamp: "2026-07-28T12:49:30Z" message: 'Created pod: sequence-graph-f7fa9-649ffd79b6-dt6lz' metadata: creationTimestamp: "2026-07-28T12:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:30Z" name: sequence-graph-f7fa9-649ffd79b6.18c67505fe2db874 namespace: kserve-ci-e2e-test resourceVersion: "17535" uid: 5bec89df-0244-46f4-9390-f760314142f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: 44c6c5b2-01f3-43fe-a07c-85f24ea54d62 kind: Event lastTimestamp: "2026-07-28T12:49:30Z" message: Scaled up replica set sequence-graph-f7fa9-649ffd79b6 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:30Z" name: sequence-graph-f7fa9.18c67505fd518ac7 namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: fb3788e0-b926-4f87-9346-23ff3b8e9bab reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17544" uid: 9331b825-cc18-4d1b-845a-9b29ab992ce2 kind: Event lastTimestamp: "2026-07-28T12:49:30Z" message: InferenceGraph [sequence-graph-f7fa9] is Ready metadata: creationTimestamp: "2026-07-28T12:49:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:49:30Z" name: sequence-graph-f7fa9.18c6750601eb94fe namespace: kserve-ci-e2e-test resourceVersion: "17546" uid: 034563c0-6bf9-4076-af55-8ccb43a20497 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: e9b4d329-ed55-4006-8e75-92bb820b13ac kind: Event lastTimestamp: "2026-07-28T12:49:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:45Z" name: sequence-graph-f7fa9.18c675097c2e8990 namespace: kserve-ci-e2e-test resourceVersion: "17670" uid: cab7fd36-27b7-484b-9ee9-4b329742c8b4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:45Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: e9b4d329-ed55-4006-8e75-92bb820b13ac kind: Event lastTimestamp: "2026-07-28T12:49:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:49:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:45Z" name: sequence-graph-f7fa9.18c675097c9b6097 namespace: kserve-ci-e2e-test resourceVersion: "17672" uid: a2e697c1-117d-4e51-9b4b-1a56f5ef0bc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:50:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: e9b4d329-ed55-4006-8e75-92bb820b13ac kind: Event lastTimestamp: "2026-07-28T12:50:00Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:50:00Z" name: sequence-graph-f7fa9.18c6750cfb492b4c namespace: kserve-ci-e2e-test resourceVersion: "17775" uid: d425243d-4e6a-49d6-9357-69ceed4a7f55 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:50:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17671" uid: e9b4d329-ed55-4006-8e75-92bb820b13ac kind: Event lastTimestamp: "2026-07-28T12:50:00Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:50:00Z" name: sequence-graph-f7fa9.18c6750cfbb054e5 namespace: kserve-ci-e2e-test resourceVersion: "17777" uid: 32ba3e5e-9518-40fe-aeba-c947fad5fe19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:58:45.786278Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-fc1de-b89b597bb-hvnqj to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c67587300eb97e namespace: kserve-ci-e2e-test resourceVersion: "21987" uid: 64d25a47-c1f9-4da7-9cf4-5296982afef4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-fc1de-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:46Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c675873e3cb591 namespace: kserve-ci-e2e-test resourceVersion: "22005" uid: fa0ad3bc-3ba5-4b51-be12-e3b5dfe975fb reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:46Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21991" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:46Z" message: Add eth0 [10.132.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:58:46Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c675876d9eaa4a namespace: kserve-ci-e2e-test resourceVersion: "22014" uid: bcc62e83-66bd-4c33-a8e5-79c3e6ec6f5e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fc1de} kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:46Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:46Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c675876ef884e1 namespace: kserve-ci-e2e-test resourceVersion: "22017" uid: b21dfbc1-bd9d-4864-beb9-5eaca7a114f1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fc1de} kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:46Z" message: 'Created container: sequence-graph-fc1de' metadata: creationTimestamp: "2026-07-28T12:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:46Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c675877209dc25 namespace: kserve-ci-e2e-test resourceVersion: "22019" uid: 1177b7f6-4301-470e-9ca2-a02cdda00e5e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fc1de} kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:46Z" message: Started container sequence-graph-fc1de metadata: creationTimestamp: "2026-07-28T12:58:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:46Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c67587727f5e95 namespace: kserve-ci-e2e-test resourceVersion: "22020" uid: c5a034e6-0b13-4b58-ba27-06af03911072 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fc1de} kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:58:55Z" message: Stopping container sequence-graph-fc1de metadata: creationTimestamp: "2026-07-28T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:55Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c6758989300aea namespace: kserve-ci-e2e-test resourceVersion: "22125" uid: 3d037f73-b519-4fcd-8e40-ed1ad6ab2fde reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-fc1de} kind: Pod name: sequence-graph-fc1de-b89b597bb-hvnqj namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: 240bd96c-2163-4b91-bf1f-422c7c36e631 kind: Event lastTimestamp: "2026-07-28T12:59:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:59:23Z" name: sequence-graph-fc1de-b89b597bb-hvnqj.18c6758a2d0cc439 namespace: kserve-ci-e2e-test resourceVersion: "22519" uid: 1d22ebcd-9de5-4092-a1f5-dd562512d0a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-fc1de-b89b597bb namespace: kserve-ci-e2e-test resourceVersion: "21979" uid: 081d97f4-9079-4393-a957-98e54d245997 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'Created pod: sequence-graph-fc1de-b89b597bb-hvnqj' metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de-b89b597bb.18c675872fb28767 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: bd7472a0-b773-46c4-85d1-45b48e907abf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21977" uid: cc718319-672c-4aa0-bbc3-43a3360b206c kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: Scaled up replica set sequence-graph-fc1de-b89b597bb from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de.18c675872ee5276a namespace: kserve-ci-e2e-test resourceVersion: "21980" uid: 68d5a8fa-5273-4298-9f23-11590321a29d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21996" uid: 7c8537f0-3758-464d-aa08-7a00a0cb86b6 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: InferenceGraph [sequence-graph-fc1de] is Ready metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de.18c6758733b4e678 namespace: kserve-ci-e2e-test resourceVersion: "21998" uid: e4065229-8d44-4240-8a33-e5c7378d8930 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 7c8537f0-3758-464d-aa08-7a00a0cb86b6 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'Failed to update status for InferenceGraph "sequence-graph-fc1de": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de.18c675873590017a namespace: kserve-ci-e2e-test resourceVersion: "22001" uid: 6edae4ec-7a65-44df-8adb-701441ed682a reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:45Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21972" uid: 7c8537f0-3758-464d-aa08-7a00a0cb86b6 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:58:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:45Z" name: sequence-graph-fc1de.18c6758735905f82 namespace: kserve-ci-e2e-test resourceVersion: "22002" uid: f58c5bbc-9d7b-4e9f-ba16-61197ee09148 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:09:31.539070Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27760" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-13bf1-6d966d6b6b-lpb8z to ip-10-0-129-244.ec2.internal metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c6761d89ec65af namespace: kserve-ci-e2e-test resourceVersion: "27765" uid: ddd06587-1898-4062-8878-f4319809f589 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:31Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27769" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:09:31Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c6761da3666e53 namespace: kserve-ci-e2e-test resourceVersion: "27781" uid: 5e34e029-9812-4cd7-9258-3cc5b3e1a661 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13bf1} kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:09:31Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c6761da4bf36da namespace: kserve-ci-e2e-test resourceVersion: "27783" uid: 526addc3-3758-429e-bea5-bb9042561607 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13bf1} kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:09:32Z" message: 'Created container: splitter-graph-13bf1' metadata: creationTimestamp: "2026-07-28T13:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:32Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c6761da7fa6570 namespace: kserve-ci-e2e-test resourceVersion: "27784" uid: ec39c234-1919-4c30-a557-9164ddd25361 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13bf1} kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:09:32Z" message: Started container splitter-graph-13bf1 metadata: creationTimestamp: "2026-07-28T13:09:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:32Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c6761da86e1050 namespace: kserve-ci-e2e-test resourceVersion: "27785" uid: 4d7e3346-339c-49f0-9f1d-6e7dc5076406 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13bf1} kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:17:46Z" message: Stopping container splitter-graph-13bf1 metadata: creationTimestamp: "2026-07-28T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:46Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c67690ba2ef969 namespace: kserve-ci-e2e-test resourceVersion: "31217" uid: 5e0fee7e-496c-4a5c-9e4e-48a17baafc6f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13bf1} kind: Pod name: splitter-graph-13bf1-6d966d6b6b-lpb8z namespace: kserve-ci-e2e-test resourceVersion: "27762" uid: 56af4104-cb55-4490-b09d-133abc0205b7 kind: Event lastTimestamp: "2026-07-28T13:18:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T13:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:18:13Z" name: splitter-graph-13bf1-6d966d6b6b-lpb8z.18c676912e669080 namespace: kserve-ci-e2e-test resourceVersion: "31452" uid: 43040d74-e7fd-4733-94f6-ecf7a2788168 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-13bf1-6d966d6b6b namespace: kserve-ci-e2e-test resourceVersion: "27756" uid: b155e43a-c8c0-4e70-8593-2e526a8672c3 kind: Event lastTimestamp: "2026-07-28T13:09:31Z" message: 'Created pod: splitter-graph-13bf1-6d966d6b6b-lpb8z' metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1-6d966d6b6b.18c6761d8987981c namespace: kserve-ci-e2e-test resourceVersion: "27761" uid: 72dbe5d4-4073-4d79-9dc5-58231a89427a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27753" uid: 3dcee715-a0de-4494-92c8-313845c96379 kind: Event lastTimestamp: "2026-07-28T13:09:31Z" message: Scaled up replica set splitter-graph-13bf1-6d966d6b6b from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1.18c6761d88e28fd7 namespace: kserve-ci-e2e-test resourceVersion: "27758" uid: 36d7b22f-306e-4ac9-8b7b-17fc117725bb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:31Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27773" uid: 8071aec0-1596-4ed8-9b37-342261bcbc84 kind: Event lastTimestamp: "2026-07-28T13:09:31Z" message: InferenceGraph [splitter-graph-13bf1] is Ready metadata: creationTimestamp: "2026-07-28T13:09:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:31Z" name: splitter-graph-13bf1.18c6761d8dbf74ec namespace: kserve-ci-e2e-test resourceVersion: "27775" uid: 081a067d-f754-4e25-988c-09615f029b72 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 0578dbda-0885-4639-9b84-88272b8198e3 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: splitter-graph-13bf1.18c67621076988dd namespace: kserve-ci-e2e-test resourceVersion: "27897" uid: e2fae59f-12ce-4b7f-ab28-23a0de5e5288 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27755" uid: 0578dbda-0885-4639-9b84-88272b8198e3 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:09:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: splitter-graph-13bf1.18c6762107ccb58b namespace: kserve-ci-e2e-test resourceVersion: "27902" uid: 08ebdf43-819b-4fc8-ae9a-c082a187b9bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27898" uid: 0578dbda-0885-4639-9b84-88272b8198e3 kind: Event lastTimestamp: "2026-07-28T13:10:01Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:10:01Z" name: splitter-graph-13bf1.18c67624865e3075 namespace: kserve-ci-e2e-test resourceVersion: "28012" uid: 8125b4e8-a445-4b50-aff8-f827f49ea749 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:10:01Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27898" uid: 0578dbda-0885-4639-9b84-88272b8198e3 kind: Event lastTimestamp: "2026-07-28T13:10:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:10:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:10:01Z" name: splitter-graph-13bf1.18c6762486c0e1d1 namespace: kserve-ci-e2e-test resourceVersion: "28015" uid: e0590f43-7b20-498e-aa92-8bba5c7ce131 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:08:21.206652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26907" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-4b3a0-85574dc76d-6696p to ip-10-0-129-244.ec2.internal metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:08:21Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760d29c8d75e namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 1a2d10bb-871c-4ecb-8d7c-2d04a44e6e18 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-4b3a0-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760d37218e40 namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: 0bf372e8-9713-490a-86e6-4833bb4a8c2a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:22Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:22Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:08:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:08:22Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760d670788af namespace: kserve-ci-e2e-test resourceVersion: "26953" uid: 9593f726-e8ea-4237-9589-6b5515d2afba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:22Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" metadata: creationTimestamp: "2026-07-28T13:08:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:22Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760d686330eb namespace: kserve-ci-e2e-test resourceVersion: "26955" uid: 60ba138b-4bed-4f08-99ce-cb37818833df reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:24Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" in 2.493s (2.493s including waiting). Image size: 218226816 bytes.' metadata: creationTimestamp: "2026-07-28T13:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:24Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760dfd011495 namespace: kserve-ci-e2e-test resourceVersion: "26983" uid: 4effe78c-e0b4-4e66-a4a8-c208a4d7019c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:24Z" message: 'Created container: splitter-graph-4b3a0' metadata: creationTimestamp: "2026-07-28T13:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:24Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760dffd943d6 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: 9fd8130a-e917-49f3-8e61-712149f9f347 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:24Z" message: Started container splitter-graph-4b3a0 metadata: creationTimestamp: "2026-07-28T13:08:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:24Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760e004c4f3f namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 303a62e0-54f7-41a1-a1aa-88739a52592e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Stopping container splitter-graph-4b3a0 metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c6760f83656454 namespace: kserve-ci-e2e-test resourceVersion: "27067" uid: 96f9deaf-02f2-4413-9afc-02c73d79160f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:08:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-4b3a0} kind: Pod name: splitter-graph-4b3a0-85574dc76d-6696p namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: d7aeb751-3d59-45e0-b0b2-1c2b0fd0b45c kind: Event lastTimestamp: "2026-07-28T13:09:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T13:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:00Z" name: splitter-graph-4b3a0-85574dc76d-6696p.18c67610990ce38a namespace: kserve-ci-e2e-test resourceVersion: "27438" uid: da9a421f-9789-4e4d-8aef-4971a5b675fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-4b3a0-85574dc76d namespace: kserve-ci-e2e-test resourceVersion: "26903" uid: ccf2b7df-c675-4bf1-9127-0cf146ddea65 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created pod: splitter-graph-4b3a0-85574dc76d-6696p' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: splitter-graph-4b3a0-85574dc76d.18c6760d29637164 namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 839e1a50-4064-48a0-b5f2-fc8c556ec628 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26900" uid: 212ce0d0-2de9-4a56-96e5-25a0b7b44b78 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Scaled up replica set splitter-graph-4b3a0-85574dc76d from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: splitter-graph-4b3a0.18c6760d28ade836 namespace: kserve-ci-e2e-test resourceVersion: "26905" uid: 98f707a4-f766-476d-afd3-456204cae2d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26922" uid: f0318452-d3a3-4994-9a46-cec656c6d183 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: InferenceGraph [splitter-graph-4b3a0] is Ready metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:21Z" name: splitter-graph-4b3a0.18c6760d2dcf6722 namespace: kserve-ci-e2e-test resourceVersion: "26925" uid: 5ed53921-8eff-45f9-a011-b3ef7c6cc26d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:08:31.478667Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760f8e0b56a4 namespace: kserve-ci-e2e-test resourceVersion: "27122" uid: 1dfd53e6-2944-4a8e-9d32-f9b495ea0e70 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27128" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fa888bb0b namespace: kserve-ci-e2e-test resourceVersion: "27166" uid: f7ff3100-3852-42a1-8e95-5ec9d0c8e438 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fa9e90280 namespace: kserve-ci-e2e-test resourceVersion: "27168" uid: cce3ef00-0e40-4492-b903-b7d520f9935b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760faca6110e namespace: kserve-ci-e2e-test resourceVersion: "27171" uid: 6556720f-bd46-45fd-8b4d-47e6c2d5e470 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fad241d17 namespace: kserve-ci-e2e-test resourceVersion: "27173" uid: 1e94ccf2-cbdf-45b9-a0c7-8a55ce4fc95d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fad2f47d0 namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: 2e81612a-8be6-46de-9830-09702d086a18 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fb05af208 namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 3ac2b978-d142-42d9-a308-1398657a25d5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:08:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:32Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fb0da509e namespace: kserve-ci-e2e-test resourceVersion: "27179" uid: 32e517c3-3b4a-4463-8fb7-10a2d2a8b44e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T13:08:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:17:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:49Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c6760fffb7d00a namespace: kserve-ci-e2e-test resourceVersion: "31277" uid: bdc94f31-0332-45b9-800f-9b08a7aa1390 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:17:46Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:46Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c67690c1b4e333 namespace: kserve-ci-e2e-test resourceVersion: "31233" uid: 85220e65-a375-4bdd-890a-b57f6a1e76e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:17:46Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:46Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c67690c1b5473e namespace: kserve-ci-e2e-test resourceVersion: "31234" uid: a5a93de9-fed2-46eb-8a8d-1a3f5dee8b2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: e53ddbe5-6d13-4fa0-9eb9-0f5f3847b280 kind: Event lastTimestamp: "2026-07-28T13:17:49Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:17:49Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m.18c67691741c9a09 namespace: kserve-ci-e2e-test resourceVersion: "31276" uid: 1dadcefc-cd6c-4d74-9ea4-01803c2faa62 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-13bf1-predictor-5c4bf9fdd5 namespace: kserve-ci-e2e-test resourceVersion: "27101" uid: 9bc48013-70ce-4e0a-b841-fcbe1a484996 kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: 'Created pod: success-200-isvc-13bf1-predictor-5c4bf9fdd5-8977m' metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor-5c4bf9fdd5.18c6760f8d693934 namespace: kserve-ci-e2e-test resourceVersion: "27115" uid: 70fc014e-53c3-4a9a-9494-967036af5598 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27093" uid: 46185341-229a-4a3a-b458-c4aefbf1f86a kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Scaled up replica set success-200-isvc-13bf1-predictor-5c4bf9fdd5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-13bf1-predictor.18c6760f8bab5d47 namespace: kserve-ci-e2e-test resourceVersion: "27104" uid: 5c2d99c9-7a0c-4a7a-8926-0528ad3a1e8b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 06824896-9d60-4df8-a3cf-8e7c13d588a5 kind: Event lastTimestamp: "2026-07-28T13:09:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:01Z" name: success-200-isvc-13bf1-predictor.18c676130ab4dc33 namespace: kserve-ci-e2e-test resourceVersion: "27442" uid: ebb382d6-b727-4a62-88fc-d282570ae028 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27095" uid: 06824896-9d60-4df8-a3cf-8e7c13d588a5 kind: Event lastTimestamp: "2026-07-28T13:09:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:01Z" name: success-200-isvc-13bf1-predictor.18c676130b1ebb35 namespace: kserve-ci-e2e-test resourceVersion: "27443" uid: 9896b405-fa8f-4d2d-a26a-ffede1338309 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: 06824896-9d60-4df8-a3cf-8e7c13d588a5 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: success-200-isvc-13bf1-predictor.18c6761a09181462 namespace: kserve-ci-e2e-test resourceVersion: "27899" uid: 3370ad02-d11d-4e55-ab8a-0c058c829fa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13bf1-predictor namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: 06824896-9d60-4df8-a3cf-8e7c13d588a5 kind: Event lastTimestamp: "2026-07-28T13:09:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:46Z" name: success-200-isvc-13bf1-predictor.18c6761a097a981e namespace: kserve-ci-e2e-test resourceVersion: "27901" uid: c8c59951-5056-483d-bc58-461379b84f8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27079" uid: 66e40fdb-0cdb-4657-9d4e-995e69bd59a4 kind: Event lastTimestamp: "2026-07-28T13:09:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:19Z" name: success-200-isvc-13bf1.18c6760f8ba71734 namespace: kserve-ci-e2e-test resourceVersion: "27622" uid: 5748e383-6e4c-415d-b1cd-5a9107ab1952 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13bf1 namespace: kserve-ci-e2e-test resourceVersion: "27620" uid: 66e40fdb-0cdb-4657-9d4e-995e69bd59a4 kind: Event lastTimestamp: "2026-07-28T13:09:19Z" message: InferenceService [success-200-isvc-13bf1] is Ready metadata: creationTimestamp: "2026-07-28T13:09:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:19Z" name: success-200-isvc-13bf1.18c6761aba8704d7 namespace: kserve-ci-e2e-test resourceVersion: "27621" uid: 835f0a6c-cdf7-4358-99e8-b89d559b3941 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:07:21.180001Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26216" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4b3a0-predictor-f958c8967-5ttpg to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff2fead3aa namespace: kserve-ci-e2e-test resourceVersion: "26222" uid: 5fc19f5e-d48f-4610-8c4b-861fb6ad396f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-4b3a0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff3c042d2e namespace: kserve-ci-e2e-test resourceVersion: "26273" uid: 5f6ed529-18ef-4884-b0c0-bba4e7641afe reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26232" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff6dfca392 namespace: kserve-ci-e2e-test resourceVersion: "26296" uid: e4aea4f4-3cc0-436d-80e5-30d56ce187f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff6f59a14a namespace: kserve-ci-e2e-test resourceVersion: "26298" uid: 8febde05-851b-4d2a-bf81-bfb1beaa8d16 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff724a40d7 namespace: kserve-ci-e2e-test resourceVersion: "26299" uid: c708d491-8918-4c5b-8699-5a4bcf31c11c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff72c6c64c namespace: kserve-ci-e2e-test resourceVersion: "26300" uid: 70e7ac5a-522a-453f-b058-07889e6ac56f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff72d14d95 namespace: kserve-ci-e2e-test resourceVersion: "26301" uid: 28ffef1c-eab9-4af4-bfa6-78504e988d2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff76c2f05f namespace: kserve-ci-e2e-test resourceVersion: "26302" uid: 0964c16e-ec8c-499e-a8f6-c100b1fb9c63 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:07:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:22Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ff7744246b namespace: kserve-ci-e2e-test resourceVersion: "26303" uid: 96846f28-28df-4c35-9cac-85088ebc63af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:07:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:07:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:59Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c675ffa09dbe35 namespace: kserve-ci-e2e-test resourceVersion: "26605" uid: 65d2b2ef-ce67-40d5-8a82-9856fefb85cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c6760f8a042a4e namespace: kserve-ci-e2e-test resourceVersion: "27087" uid: 24f3d954-a6fc-4368-a808-000f886345ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:08:31Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:31Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c6760f8a04b34a namespace: kserve-ci-e2e-test resourceVersion: "27089" uid: 89da681f-6e5c-4fc4-9aa3-75b2a7355fa2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: dedeb632-087e-4ced-bce7-d8c8a770fecb kind: Event lastTimestamp: "2026-07-28T13:08:34Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:34Z" name: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg.18c67610292546e3 namespace: kserve-ci-e2e-test resourceVersion: "27208" uid: e6111c3b-4b6e-4ac9-81cb-51180c570cb6 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4b3a0-predictor-f958c8967 namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 1d3dee40-95d4-41d0-b9da-6f52b4ad213e kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Created pod: success-200-isvc-4b3a0-predictor-f958c8967-5ttpg' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0-predictor-f958c8967.18c675ff2f7ae8ba namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 43e4b6bd-d709-4e8e-931d-09f558722d16 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: f09445f0-a742-4421-8062-5d04d68db41f kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Scaled up replica set success-200-isvc-4b3a0-predictor-f958c8967 from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0-predictor.18c675ff2be3c10c namespace: kserve-ci-e2e-test resourceVersion: "26201" uid: eb5733f9-5102-4839-98f7-dec819c32504 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 69a9d63d-0fda-4fe4-b272-6f5615cb6541 kind: Event lastTimestamp: "2026-07-28T13:07:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:36Z" name: success-200-isvc-4b3a0-predictor.18c67602aa6446ab namespace: kserve-ci-e2e-test resourceVersion: "26422" uid: ed2e2c5f-9897-4a96-8b42-f9e4ebe7adcd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: 69a9d63d-0fda-4fe4-b272-6f5615cb6541 kind: Event lastTimestamp: "2026-07-28T13:07:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:07:36Z" name: success-200-isvc-4b3a0-predictor.18c67602aabf8d15 namespace: kserve-ci-e2e-test resourceVersion: "26424" uid: f18f76f5-10c1-4386-a23a-3c83d050a1cd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 69a9d63d-0fda-4fe4-b272-6f5615cb6541 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-4b3a0-predictor.18c676062a1137de namespace: kserve-ci-e2e-test resourceVersion: "26901" uid: 05b76234-5fb7-4d61-8d45-b1389767bd0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:07:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4b3a0-predictor namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 69a9d63d-0fda-4fe4-b272-6f5615cb6541 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-4b3a0-predictor.18c676062a7dd2b1 namespace: kserve-ci-e2e-test resourceVersion: "26904" uid: ed865693-35cf-4231-ae25-46f29efd2db4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: da5cba67-42db-4800-9cfe-29696b7d2a46 kind: Event lastTimestamp: "2026-07-28T13:08:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:09Z" name: success-200-isvc-4b3a0.18c675ff2c0c1dd4 namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: cec0e646-c081-4da3-919c-066a7143bf58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: da5cba67-42db-4800-9cfe-29696b7d2a46 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'Failed to update status for InferenceService "success-200-isvc-4b3a0": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b3a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0.18c675ff2cc86480 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: f9918491-aff8-420b-8f6a-8e991b82300c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26182" uid: da5cba67-42db-4800-9cfe-29696b7d2a46 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-4b3a0": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-4b3a0.18c675ff2cc8d5be namespace: kserve-ci-e2e-test resourceVersion: "26212" uid: 8fbd749d-6504-47f1-aae7-8157e1381afc reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4b3a0 namespace: kserve-ci-e2e-test resourceVersion: "26691" uid: da5cba67-42db-4800-9cfe-29696b7d2a46 kind: Event lastTimestamp: "2026-07-28T13:08:09Z" message: InferenceService [success-200-isvc-4b3a0] is Ready metadata: creationTimestamp: "2026-07-28T13:08:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:09Z" name: success-200-isvc-4b3a0.18c6760a5c368ad4 namespace: kserve-ci-e2e-test resourceVersion: "26692" uid: db0697e9-8292-446f-8ac6-35411c9b4cf9 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:56:55.980467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20650" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756d9f1ef94a namespace: kserve-ci-e2e-test resourceVersion: "20659" uid: d4027750-f46a-4b45-b182-c24d3d9753e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-75e21-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:56:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:56Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756dad0f7a8d namespace: kserve-ci-e2e-test resourceVersion: "20704" uid: 7faff637-f04f-4462-b743-4b43177a32ce reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20671" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756dde1c55d9 namespace: kserve-ci-e2e-test resourceVersion: "20719" uid: d3716aa2-b33e-41d0-8bf3-ecaf8735fef1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756ddf8994ba namespace: kserve-ci-e2e-test resourceVersion: "20721" uid: 7136d454-1174-4974-b2a9-6c59ea68c04c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756de2bcf4ae namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: fe2c0316-0835-4d00-8f4f-88ac557b74f2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756de346a991 namespace: kserve-ci-e2e-test resourceVersion: "20726" uid: 7f72beea-b75e-4905-beb7-e7a85f6a538e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756de351cfd5 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 7ae6e86d-d83d-40ad-8c17-e270762efe11 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756de6766ff7 namespace: kserve-ci-e2e-test resourceVersion: "20732" uid: 6ccb9b70-0d3f-4a8f-be8b-d5accb9f4470 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:56:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:57Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756de702567e namespace: kserve-ci-e2e-test resourceVersion: "20733" uid: 7d95e916-7f4d-47cf-be0f-a79a8dac241a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:57:33Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:33Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6756df5228071 namespace: kserve-ci-e2e-test resourceVersion: "21026" uid: 75d2abfa-4438-4232-83c1-ba532588ea2c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6757df8e94434 namespace: kserve-ci-e2e-test resourceVersion: "21527" uid: 396570a8-a64b-48cd-a6ef-696cbb702b13 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6757df8ea05bc namespace: kserve-ci-e2e-test resourceVersion: "21529" uid: 16b0cce8-0bd4-4cac-8960-64f6c993a237 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8 namespace: kserve-ci-e2e-test resourceVersion: "20653" uid: 6c748ae6-3fed-4d4e-8cc0-57fa282a85d9 kind: Event lastTimestamp: "2026-07-28T12:58:08Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:08Z" name: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8.18c6757e7d641185 namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: 45a2ea3d-5d99-41cc-99fd-af3e7b7ec360 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-75e21-predictor-7df67ccd87 namespace: kserve-ci-e2e-test resourceVersion: "20632" uid: 3e38b0c5-fff0-40de-a72f-e6b2047f7eec kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: 'Created pod: success-200-isvc-75e21-predictor-7df67ccd87-mnfc8' metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-75e21-predictor-7df67ccd87.18c6756d9ea18011 namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: d76f1139-6468-4980-8560-70bc58070a50 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20626" uid: 7a6a3ffc-6b8e-4737-8c87-9c9424aaa35d kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: Scaled up replica set success-200-isvc-75e21-predictor-7df67ccd87 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-75e21-predictor.18c6756d9acd47ff namespace: kserve-ci-e2e-test resourceVersion: "20637" uid: 884d3434-2bca-48ca-aa0b-03f8d19a92d3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:57:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 3f59c76f-1d37-447f-ba85-c47e43c9727e kind: Event lastTimestamp: "2026-07-28T12:57:55Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:55Z" name: success-200-isvc-75e21-predictor.18c675711993fb30 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 2ac135fb-5140-48aa-8229-dd0cd7ac4d4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:57:10Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20629" uid: 3f59c76f-1d37-447f-ba85-c47e43c9727e kind: Event lastTimestamp: "2026-07-28T12:57:55Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:57:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:56Z" name: success-200-isvc-75e21-predictor.18c675711a2386f0 namespace: kserve-ci-e2e-test resourceVersion: "21374" uid: 071cf1f7-b81e-413b-ac5a-b6c86772e4c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 3f59c76f-1d37-447f-ba85-c47e43c9727e kind: Event lastTimestamp: "2026-07-28T12:57:25Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:25Z" name: success-200-isvc-75e21-predictor.18c6757498e3c60b namespace: kserve-ci-e2e-test resourceVersion: "20965" uid: 4f942cc2-5085-4e0b-82df-1568ebc62fac reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:25Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-75e21-predictor namespace: kserve-ci-e2e-test resourceVersion: "20853" uid: 3f59c76f-1d37-447f-ba85-c47e43c9727e kind: Event lastTimestamp: "2026-07-28T12:57:25Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:57:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:25Z" name: success-200-isvc-75e21-predictor.18c67574996d2ccb namespace: kserve-ci-e2e-test resourceVersion: "20967" uid: 7db72b5b-768e-4f04-9946-441b311e6c9f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 36027740-528b-4f53-89c1-8b21939add88 kind: Event lastTimestamp: "2026-07-28T12:57:43Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:43Z" name: success-200-isvc-75e21.18c6756d9ad9a191 namespace: kserve-ci-e2e-test resourceVersion: "21107" uid: 442de5d0-1a8e-44e0-8f9b-23fd8a71f980 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 36027740-528b-4f53-89c1-8b21939add88 kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: 'Failed to update status for InferenceService "success-200-isvc-75e21": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-75e21": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-75e21.18c6756d9b6b3565 namespace: kserve-ci-e2e-test resourceVersion: "20642" uid: c928290c-5b95-4b9c-bc2e-50fbcf076856 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "20616" uid: 36027740-528b-4f53-89c1-8b21939add88 kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-75e21": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-75e21.18c6756d9b6b8d96 namespace: kserve-ci-e2e-test resourceVersion: "20643" uid: 46896e29-2010-490c-b51e-22ce940a12e3 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-75e21 namespace: kserve-ci-e2e-test resourceVersion: "21105" uid: 36027740-528b-4f53-89c1-8b21939add88 kind: Event lastTimestamp: "2026-07-28T12:57:43Z" message: InferenceService [success-200-isvc-75e21] is Ready metadata: creationTimestamp: "2026-07-28T12:57:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:43Z" name: success-200-isvc-75e21.18c67578b06606da namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: c7aaae24-3e52-425b-aa91-097265c0d93c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:46:00.480986Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:46:00Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d5004eb6bc namespace: kserve-ci-e2e-test resourceVersion: "15101" uid: 1ee432ab-8e4f-4670-afb9-57af2058a837 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Add eth0 [10.134.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:46:00Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d51cf3b5ad namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: f794e603-47b6-4847-bda2-74a610b86d95 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:00Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d51eb4bf3e namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: 66d64cab-5a02-49e1-a918-9d486db45c0d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" in 17.324s (17.324s including waiting). Image size: 1334926963 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d9274ef56f namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: b6a4ac20-ad49-49ac-a1bb-68091f98a636 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d92c8f197f namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 31bb1e31-7824-437b-bcc1-1a6309ec2ffa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d92d5cb0e3 namespace: kserve-ci-e2e-test resourceVersion: "15449" uid: e2c410b8-bab3-4213-a233-a60b58ed70a4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:18Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T12:46:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:18Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674d92d6df5e3 namespace: kserve-ci-e2e-test resourceVersion: "15451" uid: 369e0bf9-219f-4993-bf73-3469d3174316 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 4.742s (4.742s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674da481c7578 namespace: kserve-ci-e2e-test resourceVersion: "15490" uid: 11b1f545-c633-407d-b711-7c7192cf25e8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674da574f4a96 namespace: kserve-ci-e2e-test resourceVersion: "15496" uid: ad3085c5-d033-4a56-84de-a869da1aa885 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:46:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:46:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:46:23Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674da59429ecc namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 9d9aff23-6186-4e6d-9e14-983deadc275d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:46:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:47:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:46:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:00Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674daa44f6052 namespace: kserve-ci-e2e-test resourceVersion: "15813" uid: 4563afe6-4543-439b-b390-54d6879423a1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674ec71c78252 namespace: kserve-ci-e2e-test resourceVersion: "16262" uid: 20012904-271d-4a57-be67-286f64ff97a3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j namespace: kserve-ci-e2e-test resourceVersion: "15098" uid: fcce6d64-f567-4c44-ade5-098cf14ea233 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j.18c674ec71c8208b namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 556a408f-11b7-48ee-9a2d-007db2bc6ebd reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9baeb-predictor-6f9fb747fc namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: f7e86f84-c71d-48b0-8a55-829b738b6b7f kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: 'Created pod: success-200-isvc-9baeb-predictor-6f9fb747fc-4td2j' metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: success-200-isvc-9baeb-predictor-6f9fb747fc.18c674d5000fae20 namespace: kserve-ci-e2e-test resourceVersion: "15099" uid: 98b954c2-f05a-40ae-9a7a-d2ce6d12aaef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: ad06c120-4e9b-4aa2-ac57-1083c64f2221 kind: Event lastTimestamp: "2026-07-28T12:46:00Z" message: Scaled up replica set success-200-isvc-9baeb-predictor-6f9fb747fc from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:00Z" name: success-200-isvc-9baeb-predictor.18c674d4fcb6082d namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: d3798a0b-ea5c-4fd1-94eb-f6631ab26989 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 850758fd-31b5-48b5-99dc-376dbdef8d38 kind: Event lastTimestamp: "2026-07-28T12:46:45Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:45Z" name: success-200-isvc-9baeb-predictor.18c674d8820267fc namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: abbb971d-b672-4aeb-b386-ed0545b785c5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:46:15Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 850758fd-31b5-48b5-99dc-376dbdef8d38 kind: Event lastTimestamp: "2026-07-28T12:46:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:46:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:46:45Z" name: success-200-isvc-9baeb-predictor.18c674d88241ca03 namespace: kserve-ci-e2e-test resourceVersion: "15695" uid: 4d0accc7-94d1-4bcc-8102-fc88427a2a99 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 850758fd-31b5-48b5-99dc-376dbdef8d38 kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: success-200-isvc-9baeb-predictor.18c674e2ff0781ff namespace: kserve-ci-e2e-test resourceVersion: "16091" uid: 6c91170b-ce11-496c-b169-7e33b4ec6f82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:47:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9baeb-predictor namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 850758fd-31b5-48b5-99dc-376dbdef8d38 kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: success-200-isvc-9baeb-predictor.18c674e2ff8a09a0 namespace: kserve-ci-e2e-test resourceVersion: "16092" uid: d20556fd-fdbe-4ea2-bb32-eae195a22306 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T12:46:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9baeb namespace: kserve-ci-e2e-test resourceVersion: "15080" uid: d85c362f-7f07-4504-9831-1b485ed2eae9 kind: Event lastTimestamp: "2026-07-28T12:47:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:46:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:10Z" name: success-200-isvc-9baeb.18c674d5027c90c6 namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: d4816ec4-a7d0-48d6-b1ed-6302a638744e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9baeb namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: d85c362f-7f07-4504-9831-1b485ed2eae9 kind: Event lastTimestamp: "2026-07-28T12:47:10Z" message: InferenceService [success-200-isvc-9baeb] is Ready metadata: creationTimestamp: "2026-07-28T12:47:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:10Z" name: success-200-isvc-9baeb.18c674e560a5b626 namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 870cc488-4c62-4431-8827-723e26176795 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:58:56.068472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d01ec-predictor-6667644fcc-v6sbh to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:58:56Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758994ec8bae namespace: kserve-ci-e2e-test resourceVersion: "22181" uid: dda06166-9c56-42a4-8c8b-5bd69e1d729c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-d01ec-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c675899f456eee namespace: kserve-ci-e2e-test resourceVersion: "22235" uid: fb71d646-b6bc-436b-8ebc-3b0d540e1577 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22188" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a08a934f7 namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 6b13d2c6-c8b3-48c7-a78a-37e3a158d8c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a09fa1be6 namespace: kserve-ci-e2e-test resourceVersion: "22253" uid: 0badf66f-5a0f-4f11-b02d-d9c116908f46 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a0de24857 namespace: kserve-ci-e2e-test resourceVersion: "22256" uid: 09b21f26-4c60-4446-850a-fb416325fde7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a0e60e441 namespace: kserve-ci-e2e-test resourceVersion: "22257" uid: 37e8dba3-41e7-465f-ba66-286b631c94ea reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a0e6bbe31 namespace: kserve-ci-e2e-test resourceVersion: "22258" uid: 1bec9593-baa7-4c4f-a02c-e3b7aee98eeb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a1143e27f namespace: kserve-ci-e2e-test resourceVersion: "22259" uid: 711cbad2-21e8-4184-aa49-3cdee897bbe8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a11cfd727 namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: b53a895b-a530-4cac-a17b-18a0789871a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:58:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T13:08:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:26Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6758a7e55a5ef namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: 99a03125-9e01-493a-a94b-83977c3b50e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6760d13a667eb namespace: kserve-ci-e2e-test resourceVersion: "26824" uid: 8fd15740-d200-4436-a22c-e54d19f75af4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6760d13ad3140 namespace: kserve-ci-e2e-test resourceVersion: "26825" uid: 9bc16abd-7e40-4fae-9c12-23294bba5b2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh namespace: kserve-ci-e2e-test resourceVersion: "22174" uid: 22af2408-38a6-4630-9afe-678a75a9954d kind: Event lastTimestamp: "2026-07-28T13:08:25Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:25Z" name: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh.18c6760d1ccad63f namespace: kserve-ci-e2e-test resourceVersion: "26999" uid: b2cb2c87-4b3a-4129-9268-a34c3304401e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d01ec-predictor-6667644fcc namespace: kserve-ci-e2e-test resourceVersion: "22159" uid: 6de497fe-cb04-4107-a0e5-92f1a117ee02 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: 'Created pod: success-200-isvc-d01ec-predictor-6667644fcc-v6sbh' metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:56Z" name: success-200-isvc-d01ec-predictor-6667644fcc.18c67589944f4655 namespace: kserve-ci-e2e-test resourceVersion: "22173" uid: 05ffbf67-cb95-447a-a668-e0c7b7bf6750 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: eb9228ec-d92f-4f01-b4a6-1056557a9d01 kind: Event lastTimestamp: "2026-07-28T12:58:56Z" message: Scaled up replica set success-200-isvc-d01ec-predictor-6667644fcc from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:56Z" name: success-200-isvc-d01ec-predictor.18c67589928d17ff namespace: kserve-ci-e2e-test resourceVersion: "22161" uid: 1725b5de-e244-4a0c-90d9-38c554932fa9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:59:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: e94028ed-ac39-49b2-a0b0-dbd581f025e2 kind: Event lastTimestamp: "2026-07-28T12:59:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:56Z" name: success-200-isvc-d01ec-predictor.18c6758d114ca29e namespace: kserve-ci-e2e-test resourceVersion: "22783" uid: 59767b0c-e6fc-40c8-8348-6977ef3eaabc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T12:59:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22158" uid: e94028ed-ac39-49b2-a0b0-dbd581f025e2 kind: Event lastTimestamp: "2026-07-28T12:59:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:59:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:56Z" name: success-200-isvc-d01ec-predictor.18c6758d11be5b13 namespace: kserve-ci-e2e-test resourceVersion: "22784" uid: 97f7ded4-4b9e-4cf6-bfe2-7a660bfab9c3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: e94028ed-ac39-49b2-a0b0-dbd581f025e2 kind: Event lastTimestamp: "2026-07-28T12:59:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:26Z" name: success-200-isvc-d01ec-predictor.18c6759090b9bc80 namespace: kserve-ci-e2e-test resourceVersion: "22537" uid: c5d4cb9f-3642-46e0-9e94-04f659c95840 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d01ec-predictor namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: e94028ed-ac39-49b2-a0b0-dbd581f025e2 kind: Event lastTimestamp: "2026-07-28T12:59:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:59:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:26Z" name: success-200-isvc-d01ec-predictor.18c675909116210f namespace: kserve-ci-e2e-test resourceVersion: "22540" uid: 8ee957c3-0af7-4ec2-8f41-445a30a6609c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22131" uid: 7e3ee69e-92ab-47f3-a931-f0b7edb1dab7 kind: Event lastTimestamp: "2026-07-28T12:59:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:59:46Z" name: success-200-isvc-d01ec.18c67589929d455f namespace: kserve-ci-e2e-test resourceVersion: "22715" uid: c78322a7-5065-4151-ada4-0c30ddefd366 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:59:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d01ec namespace: kserve-ci-e2e-test resourceVersion: "22712" uid: 7e3ee69e-92ab-47f3-a931-f0b7edb1dab7 kind: Event lastTimestamp: "2026-07-28T12:59:46Z" message: InferenceService [success-200-isvc-d01ec] is Ready metadata: creationTimestamp: "2026-07-28T12:59:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:59:46Z" name: success-200-isvc-d01ec.18c6759539b1f88f namespace: kserve-ci-e2e-test resourceVersion: "22714" uid: ef91273d-e111-4b68-bb3c-2de11357991e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:58:06.262017Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-da172-predictor-79f56c75f7-5c7vk to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757dfc3a4be4 namespace: kserve-ci-e2e-test resourceVersion: "21557" uid: 85f4bccb-243c-48e9-ac97-7332239d6e08 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-da172-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e04620c9d namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 768fe2f8-fc48-466a-9d74-8dc63b089968 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21563" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e3ae46e38 namespace: kserve-ci-e2e-test resourceVersion: "21609" uid: 782c20d2-3c14-4f24-b38b-07c0b706c517 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e3c470405 namespace: kserve-ci-e2e-test resourceVersion: "21611" uid: 086df5b7-d756-43b4-b9ba-1b8b60cb8c84 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e3fd728e4 namespace: kserve-ci-e2e-test resourceVersion: "21613" uid: 998e6e41-7631-4818-aa3b-bb8e6883c8c4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e40542969 namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: 6d7d8a29-b223-42e9-a6e5-442e5e543f58 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e4060232b namespace: kserve-ci-e2e-test resourceVersion: "21616" uid: 4a944586-bb0c-43e3-827e-d3701c5afc7a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e43bfb26a namespace: kserve-ci-e2e-test resourceVersion: "21619" uid: 87d9cafa-f273-4116-9ded-bb6839ac4781 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T12:58:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:07Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e443e9d6a namespace: kserve-ci-e2e-test resourceVersion: "21620" uid: a8ff37d8-08ae-4f19-8dd4-b712486b52ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:58:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T13:07:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:23Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c6757e54fcb926 namespace: kserve-ci-e2e-test resourceVersion: "26324" uid: a69054c5-4fd2-40fa-a3f5-6f740d003af9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c675ff2d2921f5 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: 45d65bb1-9451-407c-8547-a9fe84071ccb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T13:07:21Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:07:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:21Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c675ff2d29f4ef namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: c33d4488-1db6-434f-9222-bf500223306a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:07:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: ea8ca64f-9e38-4fd4-8e4b-dcef9bcef966 kind: Event lastTimestamp: "2026-07-28T13:07:23Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:07:23Z" name: success-200-isvc-da172-predictor-79f56c75f7-5c7vk.18c675ffc9704824 namespace: kserve-ci-e2e-test resourceVersion: "26323" uid: 6c17d201-d655-4864-b1b9-ce0d68e600d5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-da172-predictor-79f56c75f7 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: fa18ffc3-6644-4b92-bfb6-c5ff9778d074 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: 'Created pod: success-200-isvc-da172-predictor-79f56c75f7-5c7vk' metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-da172-predictor-79f56c75f7.18c6757dfbaa1b81 namespace: kserve-ci-e2e-test resourceVersion: "21551" uid: 000c369a-16e9-4eb4-95a6-2ef65f75e607 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: e8b57627-671f-4418-8dbb-c69cfa9a19d1 kind: Event lastTimestamp: "2026-07-28T12:58:06Z" message: Scaled up replica set success-200-isvc-da172-predictor-79f56c75f7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:06Z" name: success-200-isvc-da172-predictor.18c6757df9f36640 namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: 36257e26-d479-462a-ac1c-dfeeaf77f936 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 41f5991c-27d4-4e4d-8706-53f975560430 kind: Event lastTimestamp: "2026-07-28T12:58:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:21Z" name: success-200-isvc-da172-predictor.18c67581783e3ba8 namespace: kserve-ci-e2e-test resourceVersion: "21758" uid: d2de14c0-6c77-42c1-b144-a1d895e5054e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 41f5991c-27d4-4e4d-8706-53f975560430 kind: Event lastTimestamp: "2026-07-28T12:58:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:21Z" name: success-200-isvc-da172-predictor.18c6758178a11234 namespace: kserve-ci-e2e-test resourceVersion: "21760" uid: 4086f206-d496-4f75-88ee-496b5c8688a1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 41f5991c-27d4-4e4d-8706-53f975560430 kind: Event lastTimestamp: "2026-07-28T12:59:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:21Z" name: success-200-isvc-da172-predictor.18c67584f7625aa9 namespace: kserve-ci-e2e-test resourceVersion: "22498" uid: 3bd6a87e-7056-4e59-9118-70b534d10982 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:58:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-da172-predictor namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: 41f5991c-27d4-4e4d-8706-53f975560430 kind: Event lastTimestamp: "2026-07-28T12:59:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:59:21Z" name: success-200-isvc-da172-predictor.18c67584f7d0cb74 namespace: kserve-ci-e2e-test resourceVersion: "22499" uid: ec82bcf0-43f1-4f78-8811-c35b2112f691 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:58:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "21508" uid: 8c65c6d0-7597-4a44-b8cf-73798daaf89d kind: Event lastTimestamp: "2026-07-28T12:58:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:58:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:53Z" name: success-200-isvc-da172.18c6757df9f25108 namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: 40262228-9dba-4fa3-8d8d-d3c3469aece5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-da172 namespace: kserve-ci-e2e-test resourceVersion: "22087" uid: 8c65c6d0-7597-4a44-b8cf-73798daaf89d kind: Event lastTimestamp: "2026-07-28T12:58:53Z" message: InferenceService [success-200-isvc-da172] is Ready metadata: creationTimestamp: "2026-07-28T12:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:53Z" name: success-200-isvc-da172.18c675890fb60ae5 namespace: kserve-ci-e2e-test resourceVersion: "22088" uid: 19493a89-83fb-4488-a9c5-0dd67f45bbf4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:08:20.922224Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26846" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-dde97-predictor-8465f8b49c-h25g6 to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d18d4e072 namespace: kserve-ci-e2e-test resourceVersion: "26850" uid: 6e0fda43-079c-4be9-bc8e-7b0c7ee3cb8a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26854" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d33946cdf namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: c4258f88-e667-4954-95c5-69ebc83bf42d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d34e27814 namespace: kserve-ci-e2e-test resourceVersion: "26929" uid: f22e0e77-c99f-45bb-b2e9-d30ae14da2fe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d3837b0b9 namespace: kserve-ci-e2e-test resourceVersion: "26931" uid: fc3c9e28-af01-4822-b4ad-900b5acd1ae8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d38b116ca namespace: kserve-ci-e2e-test resourceVersion: "26932" uid: cfe0607d-cabb-4c1a-bd75-7c7af8b93f41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d38bd80bb namespace: kserve-ci-e2e-test resourceVersion: "26934" uid: ca5e0e23-d360-40f4-849b-66a389136b8c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d3c0a626b namespace: kserve-ci-e2e-test resourceVersion: "26938" uid: 429b16dc-9425-4617-9bbd-ffb3ec1ece76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:21Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:08:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:21Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d3c870b29 namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: f2fda35e-0923-4cab-9234-5351193fc94e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:08:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:08:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:08:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:08:58Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c6760d6cd4ddc1 namespace: kserve-ci-e2e-test resourceVersion: "27415" uid: 0f0eabd2-a506-414e-8557-06b1fd5f4504 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:25:40Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T13:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:40Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c676ff1dbe1608 namespace: kserve-ci-e2e-test resourceVersion: "34516" uid: 4a314973-c0f2-409d-a26a-689bc981b4ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:25:40Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T13:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:40Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c676ff1dbebff1 namespace: kserve-ci-e2e-test resourceVersion: "34518" uid: 9d2cef0a-a225-4d74-bccf-a81a517d8090 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:25:43Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T13:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:43Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c676ffcd733245 namespace: kserve-ci-e2e-test resourceVersion: "34587" uid: 2f80bfdd-48b6-4bfc-b857-b02b2411742b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:25:49Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-07-28T13:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:49Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c67701331f013b namespace: kserve-ci-e2e-test resourceVersion: "34642" uid: fea8e2d9-4c33-4014-bdfc-d91ea575e31a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6 namespace: kserve-ci-e2e-test resourceVersion: "26848" uid: 82a107a6-dd9a-4044-8da6-6b8013f60ad8 kind: Event lastTimestamp: "2026-07-28T13:25:49Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: i/o timeout' metadata: creationTimestamp: "2026-07-28T13:25:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:49Z" name: success-200-isvc-dde97-predictor-8465f8b49c-h25g6.18c677013368fbf7 namespace: kserve-ci-e2e-test resourceVersion: "34643" uid: efff0f16-29a7-469b-823c-14b680406666 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-dde97-predictor-8465f8b49c namespace: kserve-ci-e2e-test resourceVersion: "26837" uid: e3ad5e24-59e5-4ff2-adbc-272ad0f92cf0 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: 'Created pod: success-200-isvc-dde97-predictor-8465f8b49c-h25g6' metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-dde97-predictor-8465f8b49c.18c6760d1860d1b5 namespace: kserve-ci-e2e-test resourceVersion: "26847" uid: f4aac200-d259-4818-ab04-66bee20a96fc reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26834" uid: 704d76f9-43fb-4171-92d0-d1ce363a4fe6 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: Scaled up replica set success-200-isvc-dde97-predictor-8465f8b49c from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-dde97-predictor.18c6760d16376720 namespace: kserve-ci-e2e-test resourceVersion: "26839" uid: 3507b150-2bb4-42b2-8e48-cb41e523e680 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 4e7e4148-7495-4751-b861-193491729c23 kind: Event lastTimestamp: "2026-07-28T13:08:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:50Z" name: success-200-isvc-dde97-predictor.18c6761094a2c382 namespace: kserve-ci-e2e-test resourceVersion: "27362" uid: aa24809f-2081-44bd-8987-6ec1cd21ad31 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T13:08:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "26836" uid: 4e7e4148-7495-4751-b861-193491729c23 kind: Event lastTimestamp: "2026-07-28T13:08:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:08:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:08:50Z" name: success-200-isvc-dde97-predictor.18c6761094ea36fe namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: 1d20c9c0-ea4a-4035-9f2a-35b1449a97f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 4e7e4148-7495-4751-b861-193491729c23 kind: Event lastTimestamp: "2026-07-28T13:09:35Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:35Z" name: success-200-isvc-dde97-predictor.18c67617927b25ef namespace: kserve-ci-e2e-test resourceVersion: "27816" uid: 9fd56228-5ad8-41b5-8b72-120e327d9303 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T13:09:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-dde97-predictor namespace: kserve-ci-e2e-test resourceVersion: "27229" uid: 4e7e4148-7495-4751-b861-193491729c23 kind: Event lastTimestamp: "2026-07-28T13:09:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:35Z" name: success-200-isvc-dde97-predictor.18c6761792dd2700 namespace: kserve-ci-e2e-test resourceVersion: "27817" uid: 89e68939-a790-40ed-834e-e461ca691f64 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 7356c60c-0e79-4c9a-8913-aa6d9ba55925 kind: Event lastTimestamp: "2026-07-28T13:09:08Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:08Z" name: success-200-isvc-dde97.18c6760d1637beca namespace: kserve-ci-e2e-test resourceVersion: "27511" uid: 4704caa8-377c-4bdf-994b-cee1a89083a2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 7356c60c-0e79-4c9a-8913-aa6d9ba55925 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: 'Failed to update status for InferenceService "success-200-isvc-dde97": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dde97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-dde97.18c6760d16e08115 namespace: kserve-ci-e2e-test resourceVersion: "26841" uid: a033c74d-f1b8-48db-87b5-a6606289ce96 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:08:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "26811" uid: 7356c60c-0e79-4c9a-8913-aa6d9ba55925 kind: Event lastTimestamp: "2026-07-28T13:08:20Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-dde97": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T13:08:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:08:20Z" name: success-200-isvc-dde97.18c6760d16e0c5bf namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: cab6b20b-97e7-401a-b1ee-afcf9e258f43 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27509" uid: 7356c60c-0e79-4c9a-8913-aa6d9ba55925 kind: Event lastTimestamp: "2026-07-28T13:09:08Z" message: InferenceService [success-200-isvc-dde97] is Ready metadata: creationTimestamp: "2026-07-28T13:09:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:08Z" name: success-200-isvc-dde97.18c67618278e6a7a namespace: kserve-ci-e2e-test resourceVersion: "27510" uid: 6810e28c-a562-4f6e-b309-36fc03c9b057 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:48:20.884285Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:48:20Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f5b0ff012a namespace: kserve-ci-e2e-test resourceVersion: "16826" uid: b8b9f3e2-cff9-441d-8bb7-cf45e41de74c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:21Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:21Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f7fa9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:48:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:21Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f5bcc31e1e namespace: kserve-ci-e2e-test resourceVersion: "16874" uid: 3a8c1668-8fe8-4242-8f75-1458985122a9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f625ae8cb0 namespace: kserve-ci-e2e-test resourceVersion: "16899" uid: a07698ac-df65-415e-9be3-39abc4fa2475 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62783473b namespace: kserve-ci-e2e-test resourceVersion: "16903" uid: 4b6d93de-b70f-4a1c-970c-7df573bba162 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62a9f5721 namespace: kserve-ci-e2e-test resourceVersion: "16904" uid: 634ba3c6-a1c5-4535-9dce-997065f66599 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62b1360a6 namespace: kserve-ci-e2e-test resourceVersion: "16905" uid: 1094e748-bdca-420d-9fe2-3ecc3dadca1d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62b1d21df namespace: kserve-ci-e2e-test resourceVersion: "16906" uid: 43eddcbb-3c1b-47b0-8ccd-79c44e5ca57b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:48:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:22Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62e9cea12 namespace: kserve-ci-e2e-test resourceVersion: "16907" uid: 77d4c0e9-7b3a-43eb-938f-56abfc6781ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:48:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:23Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f62f172faf namespace: kserve-ci-e2e-test resourceVersion: "16908" uid: 520bc5e6-8320-444c-9907-9b38c36c5d60 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:48:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:49:09Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:48:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:49:09Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c674f64113f12a namespace: kserve-ci-e2e-test resourceVersion: "17350" uid: 9f288a45-84f7-4cb4-91ad-284b675bf107 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c6757932976181 namespace: kserve-ci-e2e-test resourceVersion: "21167" uid: 902c5bbc-e0ae-4a82-8341-916b7b883361 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6 namespace: kserve-ci-e2e-test resourceVersion: "16820" uid: 7bc0fda4-9a4b-42c3-adb8-d78a4d95615c kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6.18c67579329803a1 namespace: kserve-ci-e2e-test resourceVersion: "21170" uid: cc7adb66-8657-438a-a8c4-ec031d042e20 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f7fa9-predictor-7fc8bd66b9 namespace: kserve-ci-e2e-test resourceVersion: "16798" uid: ed5f9adf-0942-42c7-89da-5827d0cd71c0 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: 'Created pod: success-200-isvc-f7fa9-predictor-7fc8bd66b9-xgfn6' metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:20Z" name: success-200-isvc-f7fa9-predictor-7fc8bd66b9.18c674f5b091876c namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: 97181c8b-df2e-45bd-820a-09e6b7717826 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16794" uid: 7abccf56-da78-4cb0-a409-2ee3c2731530 kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: Scaled up replica set success-200-isvc-f7fa9-predictor-7fc8bd66b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:20Z" name: success-200-isvc-f7fa9-predictor.18c674f5ae73c3a6 namespace: kserve-ci-e2e-test resourceVersion: "16801" uid: 9ec79917-286d-4cb2-be2b-cfa54941ccb9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: dc27a420-c744-4b30-9a94-a28bb3414b3a kind: Event lastTimestamp: "2026-07-28T12:48:50Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:50Z" name: success-200-isvc-f7fa9-predictor.18c674f92d01105c namespace: kserve-ci-e2e-test resourceVersion: "17210" uid: 40964f55-9002-4d8c-9827-4d8fabbff534 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "16797" uid: dc27a420-c744-4b30-9a94-a28bb3414b3a kind: Event lastTimestamp: "2026-07-28T12:48:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:50Z" name: success-200-isvc-f7fa9-predictor.18c674f92d6ccecc namespace: kserve-ci-e2e-test resourceVersion: "17211" uid: 3363ee1a-2b1f-4902-aefd-797ae41dac7e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: dc27a420-c744-4b30-9a94-a28bb3414b3a kind: Event lastTimestamp: "2026-07-28T12:49:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:50Z" name: success-200-isvc-f7fa9-predictor.18c675002b82189f namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: facacd22-a92a-4be9-b049-72f5d9157572 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:49:05Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f7fa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "17047" uid: dc27a420-c744-4b30-9a94-a28bb3414b3a kind: Event lastTimestamp: "2026-07-28T12:49:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:49:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:50Z" name: success-200-isvc-f7fa9-predictor.18c675002bee69d1 namespace: kserve-ci-e2e-test resourceVersion: "17708" uid: ec6224ff-7637-40dc-afbc-20af96ae4c27 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:48:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "16774" uid: b34e725d-fdb0-4fc3-b74a-b9c550e0e555 kind: Event lastTimestamp: "2026-07-28T12:49:19Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:48:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:49:19Z" name: success-200-isvc-f7fa9.18c674f5aeaf327f namespace: kserve-ci-e2e-test resourceVersion: "17441" uid: b2e66df4-1f25-4ffb-b782-88617928fb8b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:49:19Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f7fa9 namespace: kserve-ci-e2e-test resourceVersion: "17439" uid: b34e725d-fdb0-4fc3-b74a-b9c550e0e555 kind: Event lastTimestamp: "2026-07-28T12:49:19Z" message: InferenceService [success-200-isvc-f7fa9] is Ready metadata: creationTimestamp: "2026-07-28T12:49:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:49:19Z" name: success-200-isvc-f7fa9.18c6750351338e08 namespace: kserve-ci-e2e-test resourceVersion: "17440" uid: 9d0d8253-0c2c-4de8-99b3-587037f59e51 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:57:45.770252Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21199" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6757936d2d375 namespace: kserve-ci-e2e-test resourceVersion: "21206" uid: f8ff9d8a-6a13-4955-961e-d1286c7d6160 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fc1de-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c675794604c155 namespace: kserve-ci-e2e-test resourceVersion: "21246" uid: 8cd2fdcc-7d9b-463f-a8ff-1bf7443bb4e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21213" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c675797469887e namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: b737a030-02a6-44d0-9287-50f7d9680c81 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6757975d93c8d namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: fbce3735-d0e9-4db0-83b0-cb3a8fb91867 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6757978c120e2 namespace: kserve-ci-e2e-test resourceVersion: "21261" uid: b269171c-35db-4c6b-9d70-f001e8d1c209 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6757979323a77 namespace: kserve-ci-e2e-test resourceVersion: "21262" uid: b6b57ad1-eb15-48b1-9e52-a5abbc0f527e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c67579793c5665 namespace: kserve-ci-e2e-test resourceVersion: "21263" uid: 3063e97d-b2f8-4946-a286-54399e8c9ac0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c675797c6b5542 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: 5edc16c1-61ab-47a0-97f1-49280e2af898 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:57:46Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:57:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:46Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c675797ce9b57a namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 713b8c31-327f-493a-a34e-8f03375069b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:57:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:58:23Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:23Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c67579a6c6058e namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: dfa95f1c-6ae3-48ed-8f86-a20a07334052 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:58:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:58:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:55Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6758990845155 namespace: kserve-ci-e2e-test resourceVersion: "22144" uid: ba041391-26fd-49be-ae4b-43b2d7a3e12b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:58:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:58:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:56Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6758990850d88 namespace: kserve-ci-e2e-test resourceVersion: "22146" uid: 819681ed-9cd9-4eb4-a7ab-c9beab20de61 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6 namespace: kserve-ci-e2e-test resourceVersion: "21202" uid: 78b47d4c-aba1-41e4-ab30-fed27babeb3b kind: Event lastTimestamp: "2026-07-28T12:58:58Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:58:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:58:58Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6.18c6758a2ee30d5f namespace: kserve-ci-e2e-test resourceVersion: "22264" uid: e27a0966-722c-445d-9266-090126b209b1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fc1de-predictor-fb95f5f9c namespace: kserve-ci-e2e-test resourceVersion: "21184" uid: 1c2292cc-28ce-4969-a6db-5a271a18b8b2 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'Created pod: success-200-isvc-fc1de-predictor-fb95f5f9c-nwhh6' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-fc1de-predictor-fb95f5f9c.18c6757936609312 namespace: kserve-ci-e2e-test resourceVersion: "21203" uid: 75266344-e2de-4852-b3db-c9129a346c4a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 608c2d62-9120-45e9-9072-ad7e282a4e7b kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: Scaled up replica set success-200-isvc-fc1de-predictor-fb95f5f9c from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-fc1de-predictor.18c67579346a3edb namespace: kserve-ci-e2e-test resourceVersion: "21187" uid: ea501645-67b0-463f-8795-e79b2e708a35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: c154fd98-f1d7-4cac-b208-7fdf3fd0b031 kind: Event lastTimestamp: "2026-07-28T12:58:15Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:15Z" name: success-200-isvc-fc1de-predictor.18c6757cb2ecf986 namespace: kserve-ci-e2e-test resourceVersion: "21711" uid: 28edbb9d-608d-49f8-8bc4-6fff2ad1da19 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:00Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21183" uid: c154fd98-f1d7-4cac-b208-7fdf3fd0b031 kind: Event lastTimestamp: "2026-07-28T12:58:15Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:58:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:15Z" name: success-200-isvc-fc1de-predictor.18c6757cb35c12ae namespace: kserve-ci-e2e-test resourceVersion: "21712" uid: c8a9c146-a3d1-4e0a-ae34-8ef9c8a00750 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: c154fd98-f1d7-4cac-b208-7fdf3fd0b031 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: success-200-isvc-fc1de-predictor.18c67583b13b4016 namespace: kserve-ci-e2e-test resourceVersion: "21989" uid: d2f86662-ee78-4183-9055-42921e27e8bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:58:30Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc1de-predictor namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: c154fd98-f1d7-4cac-b208-7fdf3fd0b031 kind: Event lastTimestamp: "2026-07-28T12:58:45Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:58:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:58:45Z" name: success-200-isvc-fc1de-predictor.18c67583b1a0d417 namespace: kserve-ci-e2e-test resourceVersion: "21992" uid: 912be2ba-8969-4b1b-b3d0-391cb4d4d08d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: d6672547-83ef-46e4-bea1-8ccebcebb396 kind: Event lastTimestamp: "2026-07-28T12:58:33Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:33Z" name: success-200-isvc-fc1de.18c675793478395b namespace: kserve-ci-e2e-test resourceVersion: "21868" uid: 185b1f98-0d4b-4d45-8708-f843ca60d89f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: d6672547-83ef-46e4-bea1-8ccebcebb396 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'Failed to update status for InferenceService "success-200-isvc-fc1de": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-fc1de.18c67579351bd7d3 namespace: kserve-ci-e2e-test resourceVersion: "21194" uid: 47e81d47-3c4c-42dc-a69c-3aa8f76f7b96 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:57:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: d6672547-83ef-46e4-bea1-8ccebcebb396 kind: Event lastTimestamp: "2026-07-28T12:57:45Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc1de": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:57:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:57:45Z" name: success-200-isvc-fc1de.18c67579351c4fd6 namespace: kserve-ci-e2e-test resourceVersion: "21197" uid: d1c5de5b-9bdb-414c-a552-06d619465862 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:58:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc1de namespace: kserve-ci-e2e-test resourceVersion: "21863" uid: d6672547-83ef-46e4-bea1-8ccebcebb396 kind: Event lastTimestamp: "2026-07-28T12:58:33Z" message: InferenceService [success-200-isvc-fc1de] is Ready metadata: creationTimestamp: "2026-07-28T12:58:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:58:33Z" name: success-200-isvc-fc1de.18c67584633094fd namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: 643a1cf7-e151-4822-a5f6-5b45caa74550 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:47:41.107453Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-fc405-predictor-664b4c95f9-mptpw to ip-10-0-134-201.ec2.internal metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ec6e1cc1d9 namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: 820cb11f-7454-4066-841f-c4ede51f0be9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-fc405-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ec7b4e75e1 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: 33be09c5-1404-45b6-8161-2ae26c641483 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16257" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecac3bd821 namespace: kserve-ci-e2e-test resourceVersion: "16317" uid: bf07ebeb-525e-49c5-acc8-90fa64e44f7f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1146" already present on machine metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecae314871 namespace: kserve-ci-e2e-test resourceVersion: "16319" uid: 98e264d7-3554-4e03-971c-e9512ef4f10e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecb111bb15 namespace: kserve-ci-e2e-test resourceVersion: "16320" uid: e49f6a4b-3fcf-4d14-aca2-11256898a9f3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecb1997242 namespace: kserve-ci-e2e-test resourceVersion: "16321" uid: 3e03804f-7f7d-463f-a1ce-f35dd84f523e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecb1a4fbf8 namespace: kserve-ci-e2e-test resourceVersion: "16322" uid: f9fabcb4-05f2-43c4-bbdf-4dbf434edd9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecb4ba55ed namespace: kserve-ci-e2e-test resourceVersion: "16329" uid: 23014784-413d-4b1e-b48d-6f8e4625a921 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:47:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:47:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:42Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ecb526cca6 namespace: kserve-ci-e2e-test resourceVersion: "16330" uid: d537bce3-3961-47e3-8512-9ddce5b41e1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T12:47:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:48:20Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T12:47:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:20Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c674ece4ba66b3 namespace: kserve-ci-e2e-test resourceVersion: "16732" uid: a4911a48-c1f7-4140-938a-ad791ca4de17 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c6756d9bc5d0a5 namespace: kserve-ci-e2e-test resourceVersion: "20644" uid: 80bef981-df68-441a-a79d-8fd133a4d1cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: 5ed5813f-1518-42fa-aeaf-a6352bc37feb kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:55Z" name: success-200-isvc-fc405-predictor-664b4c95f9-mptpw.18c6756d9bc63fc1 namespace: kserve-ci-e2e-test resourceVersion: "20646" uid: d379a36a-c156-447c-9960-9e47ad4355c3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-201.ec2.internal source: component: kubelet host: ip-10-0-134-201.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-fc405-predictor-664b4c95f9 namespace: kserve-ci-e2e-test resourceVersion: "16222" uid: 3ee4f1bd-4151-43cf-a8b8-7694cee9ff57 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'Created pod: success-200-isvc-fc405-predictor-664b4c95f9-mptpw' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405-predictor-664b4c95f9.18c674ec6d289d2f namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 4598ea5f-ecc3-44e4-b650-287b2c398687 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16220" uid: 42d3d660-c33d-4597-a585-0e81de062ec5 kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: Scaled up replica set success-200-isvc-fc405-predictor-664b4c95f9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405-predictor.18c674ec68e15572 namespace: kserve-ci-e2e-test resourceVersion: "16223" uid: e78ce664-b4a7-4088-b04f-520c46956fbf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:47:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: dc35f2d7-125b-4973-ad98-c538d320c948 kind: Event lastTimestamp: "2026-07-28T12:48:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:56Z" name: success-200-isvc-fc405-predictor.18c674efe7bcb1e5 namespace: kserve-ci-e2e-test resourceVersion: "17253" uid: 7b791ddd-4680-481d-9f78-21782e88611e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T12:47:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16221" uid: dc35f2d7-125b-4973-ad98-c538d320c948 kind: Event lastTimestamp: "2026-07-28T12:48:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T12:47:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:56Z" name: success-200-isvc-fc405-predictor.18c674efe83ae54b namespace: kserve-ci-e2e-test resourceVersion: "17254" uid: bacda453-d4ec-440f-9c44-7b66ddffb71b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: dc35f2d7-125b-4973-ad98-c538d320c948 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:11Z" name: success-200-isvc-fc405-predictor.18c674f366fd2df2 namespace: kserve-ci-e2e-test resourceVersion: "16637" uid: 209b3e07-81c4-4a64-a4aa-fbd9347068d0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-fc405-predictor namespace: kserve-ci-e2e-test resourceVersion: "16489" uid: dc35f2d7-125b-4973-ad98-c538d320c948 kind: Event lastTimestamp: "2026-07-28T12:48:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:11Z" name: success-200-isvc-fc405-predictor.18c674f3676ebb7b namespace: kserve-ci-e2e-test resourceVersion: "16639" uid: 88b63641-a0dd-448a-8e94-1cab4a10b0e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 812e9afe-25cf-4a2c-bceb-75e798dbfb4b kind: Event lastTimestamp: "2026-07-28T12:48:30Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:30Z" name: success-200-isvc-fc405.18c674ec68f25e46 namespace: kserve-ci-e2e-test resourceVersion: "17008" uid: 4b123bd7-3f0a-4720-bb8a-f8bf6399ef31 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 812e9afe-25cf-4a2c-bceb-75e798dbfb4b kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-fc405": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc405": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405.18c674ec69a0b1e6 namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: 6c74a300-b1a5-4a56-9a98-aac607a6beb6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "16212" uid: 812e9afe-25cf-4a2c-bceb-75e798dbfb4b kind: Event lastTimestamp: "2026-07-28T12:47:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-fc405": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T12:47:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:41Z" name: success-200-isvc-fc405.18c674ec69a10913 namespace: kserve-ci-e2e-test resourceVersion: "16227" uid: 7ed614e2-7a7c-4b26-9bfa-4ca7dff77fa0 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:30Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-fc405 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: 812e9afe-25cf-4a2c-bceb-75e798dbfb4b kind: Event lastTimestamp: "2026-07-28T12:48:30Z" message: InferenceService [success-200-isvc-fc405] is Ready metadata: creationTimestamp: "2026-07-28T12:48:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:30Z" name: success-200-isvc-fc405.18c674f7db646feb namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: 69c69c91-d298-4fc9-8155-c140e3fd50d8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:47:30.556533Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16077" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9baeb-7554d76cc8-9rcpx to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:47:30Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:47:30Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674e9f93a7468 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 1a6e1b30-7509-49ad-98b2-fd2c00ac6192 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:30Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9baeb-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T12:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:30Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674ea053fe5ea namespace: kserve-ci-e2e-test resourceVersion: "16096" uid: 6d285b6a-0f01-4a62-8007-6a950d077e78 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:31Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:31Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:47:31Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674ea36ad3478 namespace: kserve-ci-e2e-test resourceVersion: "16115" uid: 2a1226b3-647d-4226-ac65-b91d0f32c5b0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:31Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" metadata: creationTimestamp: "2026-07-28T12:47:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:31Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674ea38a9396e namespace: kserve-ci-e2e-test resourceVersion: "16117" uid: 92559a8e-9d1b-43e0-9f4c-d7859f5f40d7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:34Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" in 2.438s (2.438s including waiting). Image size: 218226816 bytes.' metadata: creationTimestamp: "2026-07-28T12:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:34Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674eaca082390 namespace: kserve-ci-e2e-test resourceVersion: "16134" uid: 643f0b22-1f87-47ae-a72a-d03e335d705b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:34Z" message: 'Created container: switch-graph-9baeb' metadata: creationTimestamp: "2026-07-28T12:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:34Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674eaccd6d785 namespace: kserve-ci-e2e-test resourceVersion: "16136" uid: eb70452e-4e56-41d2-bec1-878a2543b79e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:34Z" message: Started container switch-graph-9baeb metadata: creationTimestamp: "2026-07-28T12:47:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:34Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674eacd3a30ea namespace: kserve-ci-e2e-test resourceVersion: "16137" uid: 879b7587-4df8-42be-a139-e5f6766c3593 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:47:40Z" message: Stopping container switch-graph-9baeb metadata: creationTimestamp: "2026-07-28T12:47:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:47:40Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674ec5f57ca6b namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: 6c1a7498-d4af-439b-b22b-e697d9ddfb4d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:47:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9baeb} kind: Pod name: switch-graph-9baeb-7554d76cc8-9rcpx namespace: kserve-ci-e2e-test resourceVersion: "16078" uid: 9578c1e6-0abc-4744-a9e9-d1078048a18b kind: Event lastTimestamp: "2026-07-28T12:48:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:47:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:10Z" name: switch-graph-9baeb-7554d76cc8-9rcpx.18c674ed7924a8c3 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 5b4c7ca0-7179-4ee7-aa31-14de16e34a12 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:30Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9baeb-7554d76cc8 namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: 38fd3ed3-5a6a-4bcf-bfca-d18c3a8261ea kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: 'Created pod: switch-graph-9baeb-7554d76cc8-9rcpx' metadata: creationTimestamp: "2026-07-28T12:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: switch-graph-9baeb-7554d76cc8.18c674e9f8d9b808 namespace: kserve-ci-e2e-test resourceVersion: "16079" uid: c6d6dbd6-513f-4216-8c58-d939e0da826e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:30Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9baeb namespace: kserve-ci-e2e-test resourceVersion: "16072" uid: 6b373556-b057-4398-b3be-f30112ac3f73 kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: Scaled up replica set switch-graph-9baeb-7554d76cc8 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:47:30Z" name: switch-graph-9baeb.18c674e9f7f98c27 namespace: kserve-ci-e2e-test resourceVersion: "16075" uid: f59ce9e4-551d-4ccd-b3bb-d9323f6359fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:47:30Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9baeb namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 8d0d557f-18db-4627-9c1c-efc8adb8c5fc kind: Event lastTimestamp: "2026-07-28T12:47:30Z" message: InferenceGraph [switch-graph-9baeb] is Ready metadata: creationTimestamp: "2026-07-28T12:47:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:47:30Z" name: switch-graph-9baeb.18c674e9fd2b9e91 namespace: kserve-ci-e2e-test resourceVersion: "16090" uid: 02874ea2-314e-4b6d-acd7-16a03417f3e7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T13:09:20.965839Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27646" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-dde97-5c79545b7c-hs55f to ip-10-0-129-244.ec2.internal metadata: creationTimestamp: "2026-07-28T13:09:20Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T13:09:20Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c6761b13b5a9af namespace: kserve-ci-e2e-test resourceVersion: "27651" uid: a96a4343-d4ec-4caf-9e07-6edf42d93f14 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:21Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27655" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:09:21Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T13:09:21Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c6761b2da781ec namespace: kserve-ci-e2e-test resourceVersion: "27668" uid: cd679e4b-65bf-4525-a486-c8f69854abf2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dde97} kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:09:21Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:21Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c6761b2ef492f3 namespace: kserve-ci-e2e-test resourceVersion: "27670" uid: 72baf25f-a0f5-49e8-875b-c520a63f6368 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dde97} kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:09:21Z" message: 'Created container: switch-graph-dde97' metadata: creationTimestamp: "2026-07-28T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:21Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c6761b32154aca namespace: kserve-ci-e2e-test resourceVersion: "27672" uid: afed4fa3-4c10-40c5-a4a9-bd66d499a841 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:21Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dde97} kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:09:21Z" message: Started container switch-graph-dde97 metadata: creationTimestamp: "2026-07-28T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:09:21Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c6761b328955d4 namespace: kserve-ci-e2e-test resourceVersion: "27673" uid: e0f83245-e9ce-49b8-bfe1-97c98e741a26 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:25:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dde97} kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:25:40Z" message: Stopping container switch-graph-dde97 metadata: creationTimestamp: "2026-07-28T13:25:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:25:40Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c676ff16ea8c6a namespace: kserve-ci-e2e-test resourceVersion: "34498" uid: b0011e56-51db-4754-beb8-ecd132b411d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T13:25:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-dde97} kind: Pod name: switch-graph-dde97-5c79545b7c-hs55f namespace: kserve-ci-e2e-test resourceVersion: "27648" uid: 89fea4f5-204d-4fc4-b50e-82533c39e8ec kind: Event lastTimestamp: "2026-07-28T13:26:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T13:25:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T13:26:08Z" name: switch-graph-dde97-5c79545b7c-hs55f.18c676ffc4238cdf namespace: kserve-ci-e2e-test resourceVersion: "34772" uid: bd6c0c66-ce76-4c44-9484-1ab20d82d4e8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-129-244.ec2.internal source: component: kubelet host: ip-10-0-129-244.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:20Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-dde97-5c79545b7c namespace: kserve-ci-e2e-test resourceVersion: "27643" uid: 20eb62c3-07b8-4f05-92fa-6f56e08b53a7 kind: Event lastTimestamp: "2026-07-28T13:09:20Z" message: 'Created pod: switch-graph-dde97-5c79545b7c-hs55f' metadata: creationTimestamp: "2026-07-28T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:20Z" name: switch-graph-dde97-5c79545b7c.18c6761b13470f6a namespace: kserve-ci-e2e-test resourceVersion: "27647" uid: 2f017695-cd37-4023-9ddc-aa3d1d0a1e11 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:20Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: d789a5e3-37d8-484e-bd43-cfd0758c283f kind: Event lastTimestamp: "2026-07-28T13:09:20Z" message: Scaled up replica set switch-graph-dde97-5c79545b7c from 0 to 1 metadata: creationTimestamp: "2026-07-28T13:09:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:20Z" name: switch-graph-dde97.18c6761b125fe21e namespace: kserve-ci-e2e-test resourceVersion: "27644" uid: c4eeab80-8472-4832-aa94-352626d20d98 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:21Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27658" uid: db959336-9fae-4937-a3a5-70b87bbcded4 kind: Event lastTimestamp: "2026-07-28T13:09:21Z" message: InferenceGraph [switch-graph-dde97] is Ready metadata: creationTimestamp: "2026-07-28T13:09:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T13:09:21Z" name: switch-graph-dde97.18c6761b17714d08 namespace: kserve-ci-e2e-test resourceVersion: "27660" uid: 67665f1e-46fd-4922-850c-e37c95c8c1b9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27641" uid: 2cd973b1-0c22-4820-b996-28be5c61fc2f kind: Event lastTimestamp: "2026-07-28T13:09:35Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:35Z" name: switch-graph-dde97.18c6761e90cbf16a namespace: kserve-ci-e2e-test resourceVersion: "27819" uid: 2d8f415d-7059-4c49-9bdc-820279c1b939 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:35Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27641" uid: 2cd973b1-0c22-4820-b996-28be5c61fc2f kind: Event lastTimestamp: "2026-07-28T13:09:35Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T13:09:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:35Z" name: switch-graph-dde97.18c6761e9125e5fa namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: 001f1f14-e97b-4719-96c4-eec58a30acc4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 2cd973b1-0c22-4820-b996-28be5c61fc2f kind: Event lastTimestamp: "2026-07-28T13:09:50Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:50Z" name: switch-graph-dde97.18c676220fc41f6a namespace: kserve-ci-e2e-test resourceVersion: "27936" uid: 9c835504-ce9c-46be-a638-b5a8b85db095 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T13:09:50Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-dde97 namespace: kserve-ci-e2e-test resourceVersion: "27818" uid: 2cd973b1-0c22-4820-b996-28be5c61fc2f kind: Event lastTimestamp: "2026-07-28T13:09:50Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T13:09:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T13:09:50Z" name: switch-graph-dde97.18c676221008bbbe namespace: kserve-ci-e2e-test resourceVersion: "27938" uid: 970dd1b0-611f-4f13-9d09-82dad006b613 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T12:48:41.115182Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17101" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-fc405-8559d54778-7t8j8 to ip-10-0-137-92.ec2.internal metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778-7t8j8.18c674fa66da052f namespace: kserve-ci-e2e-test resourceVersion: "17106" uid: 6c9a2910-d586-4a45-b677-91d61ac5f48b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-7bdd7bbd6-42frg source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17110" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: Add eth0 [10.132.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778-7t8j8.18c674fa80f77c60 namespace: kserve-ci-e2e-test resourceVersion: "17123" uid: a5e86218-66de-4cd8-b444-d5eb6c87eb0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc405} kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:310f41f1c7eb1a56fda87465241323f4eb2e286e2805ba71b8015ffcf6fc5d8d" already present on machine metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778-7t8j8.18c674fa82b8fad6 namespace: kserve-ci-e2e-test resourceVersion: "17125" uid: 19b1d3a2-b8b4-4827-a5fd-9639d418cf04 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc405} kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: 'Created container: switch-graph-fc405' metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778-7t8j8.18c674fa861516ee namespace: kserve-ci-e2e-test resourceVersion: "17127" uid: 569e076b-b0c4-41b7-8edb-90af3aa3829e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc405} kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: Started container switch-graph-fc405 metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778-7t8j8.18c674fa86925af6 namespace: kserve-ci-e2e-test resourceVersion: "17128" uid: a19466c0-fd45-4857-8d6b-9d2d61cc44cf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:56:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc405} kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:56:55Z" message: Stopping container switch-graph-fc405 metadata: creationTimestamp: "2026-07-28T12:56:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:56:55Z" name: switch-graph-fc405-8559d54778-7t8j8.18c6756d9353d2a6 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: b648b81c-d43f-415c-aca3-02c6e3392a94 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T12:56:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-fc405} kind: Pod name: switch-graph-fc405-8559d54778-7t8j8 namespace: kserve-ci-e2e-test resourceVersion: "17104" uid: 0510bf3d-66a3-4f0f-8036-55880df89546 kind: Event lastTimestamp: "2026-07-28T12:57:22Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T12:56:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T12:57:22Z" name: switch-graph-fc405-8559d54778-7t8j8.18c6756e119b92dd namespace: kserve-ci-e2e-test resourceVersion: "20941" uid: c7bf9bad-d0be-4482-8556-6f7b08ef1435 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-137-92.ec2.internal source: component: kubelet host: ip-10-0-137-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-fc405-8559d54778 namespace: kserve-ci-e2e-test resourceVersion: "17096" uid: bfd7d81b-bf3a-4b52-97f9-e2d38167e2d9 kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: 'Created pod: switch-graph-fc405-8559d54778-7t8j8' metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405-8559d54778.18c674fa6675a654 namespace: kserve-ci-e2e-test resourceVersion: "17103" uid: 81adcba9-ae99-4eb4-a8b3-6f861b0cddc0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-fc405 namespace: kserve-ci-e2e-test resourceVersion: "17094" uid: 680051ed-5c2c-4d6d-9b6e-ef9a116e4fab kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: Scaled up replica set switch-graph-fc405-8559d54778 from 0 to 1 metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405.18c674fa6598c49c namespace: kserve-ci-e2e-test resourceVersion: "17098" uid: 787ce086-2287-4195-b417-b646aa528914 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T12:48:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-fc405 namespace: kserve-ci-e2e-test resourceVersion: "17112" uid: c4c50a0a-f8d7-437e-aa0f-a56240a0a46a kind: Event lastTimestamp: "2026-07-28T12:48:41Z" message: InferenceGraph [switch-graph-fc405] is Ready metadata: creationTimestamp: "2026-07-28T12:48:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T12:48:41Z" name: switch-graph-fc405.18c674fa6a5c28fe namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: a99a41fc-bed3-46e2-9b5c-1bcd9efc8476 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc405 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 39a36a86-fff4-4b67-b537-424df48f5262 kind: Event lastTimestamp: "2026-07-28T12:49:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:11Z" name: switch-graph-fc405.18c674fde4009bc1 namespace: kserve-ci-e2e-test resourceVersion: "17373" uid: 0e3d942e-6bc8-4d1a-954c-599702dba98c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T12:48:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-fc405 namespace: kserve-ci-e2e-test resourceVersion: "17095" uid: 39a36a86-fff4-4b67-b537-424df48f5262 kind: Event lastTimestamp: "2026-07-28T12:49:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T12:48:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T12:49:11Z" name: switch-graph-fc405.18c674fde4780aef namespace: kserve-ci-e2e-test resourceVersion: "17374" uid: c1d444d9-3779-4fdb-93a6-5faa40d32eb1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34778"