--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:02.578220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19721" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-50a5e-dbdd56cbb-hcvhd to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:02Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c354f08edc9e namespace: kserve-ci-e2e-test resourceVersion: "19728" uid: ff4ae5aa-e449-4a0e-ac01-92840d764f69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-50a5e-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:02Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c354f87d33d7 namespace: kserve-ci-e2e-test resourceVersion: "19740" uid: 54fe1714-7453-4152-b43c-1e6b194715b8 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:03Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19730" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:03Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:03Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c3552d714295 namespace: kserve-ci-e2e-test resourceVersion: "19748" uid: df58cd64-335a-4d71-a105-d9e3524ef333 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-50a5e} kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:03Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:03Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c3552eb2f2dc namespace: kserve-ci-e2e-test resourceVersion: "19753" uid: dd12819a-5b91-4994-98ba-304ff2e92c32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-50a5e} kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:03Z" message: 'Created container: ensemble-graph-50a5e' metadata: creationTimestamp: "2026-04-22T19:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:03Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c35533897d8c namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: fa1e1aed-6b5b-47d8-bc0f-4f11c341dd55 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-50a5e} kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:03Z" message: Started container ensemble-graph-50a5e metadata: creationTimestamp: "2026-04-22T19:07:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:03Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c3553400fb9b namespace: kserve-ci-e2e-test resourceVersion: "19756" uid: 080c79f6-b45d-4acb-a328-ab7781146798 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-50a5e} kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Stopping container ensemble-graph-50a5e metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:12Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c3574857638e namespace: kserve-ci-e2e-test resourceVersion: "19849" uid: 4189b4fd-f180-43a1-8f6f-418f19feb26f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-50a5e} kind: Pod name: ensemble-graph-50a5e-dbdd56cbb-hcvhd namespace: kserve-ci-e2e-test resourceVersion: "19724" uid: 8251f818-24d3-40fb-9348-f235eaa96931 kind: Event lastTimestamp: "2026-04-22T19:07:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:40Z" name: ensemble-graph-50a5e-dbdd56cbb-hcvhd.18a8c357de336730 namespace: kserve-ci-e2e-test resourceVersion: "20256" uid: 2a2bc3b7-e278-455c-a782-99d6bf9dc4d4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-50a5e-dbdd56cbb namespace: kserve-ci-e2e-test resourceVersion: "19717" uid: 49a29f76-2f35-4936-94f7-9f3103692d5f kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'Created pod: ensemble-graph-50a5e-dbdd56cbb-hcvhd' metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: ensemble-graph-50a5e-dbdd56cbb.18a8c354ef60bed1 namespace: kserve-ci-e2e-test resourceVersion: "19722" uid: 5477e838-11e1-4dfc-92a6-58f15f637383 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-50a5e namespace: kserve-ci-e2e-test resourceVersion: "19714" uid: f27c10b1-8853-4e02-b43a-f59a0a2bd030 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: Scaled up replica set ensemble-graph-50a5e-dbdd56cbb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: ensemble-graph-50a5e.18a8c354ee894c1a namespace: kserve-ci-e2e-test resourceVersion: "19719" uid: 657ed2cc-25f3-4c94-8b78-88c52985a5c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-50a5e namespace: kserve-ci-e2e-test resourceVersion: "19735" uid: bba0ad22-0f3c-4ba4-83c4-7de4e1924ad3 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: InferenceGraph [ensemble-graph-50a5e] is Ready metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:02Z" name: ensemble-graph-50a5e.18a8c354f626e3cd namespace: kserve-ci-e2e-test resourceVersion: "19737" uid: a6dc5ad6-2d2d-466d-b8a0-71730c31ad24 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:08:12.829892Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20695" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-f6fce-77754d84cf-hl57v to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:08:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:08:12Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3654be25514 namespace: kserve-ci-e2e-test resourceVersion: "20703" uid: f1d5d3c5-fffc-45b1-b63c-4a66faa658a4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:08:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-f6fce-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:13Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3655666bf20 namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: c125a0cd-8d39-4a58-bd4e-a4771b2f396b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:13Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20705" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:08:13Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:08:13Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c36588fd86f5 namespace: kserve-ci-e2e-test resourceVersion: "20725" uid: 08180642-42ff-44f0-98cc-8f2bfa9706be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6fce} kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:08:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:13Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3658a29a045 namespace: kserve-ci-e2e-test resourceVersion: "20727" uid: 04b1bf80-ad82-4db7-8f72-6bc833abb877 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6fce} kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:08:13Z" message: 'Created container: ensemble-graph-f6fce' metadata: creationTimestamp: "2026-04-22T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:13Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3658d410652 namespace: kserve-ci-e2e-test resourceVersion: "20728" uid: b6e14945-6f1f-4ed7-9ecf-9d779f87900c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6fce} kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:08:13Z" message: Started container ensemble-graph-f6fce metadata: creationTimestamp: "2026-04-22T19:08:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:13Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3658db1dcce namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 9586ed83-f67c-4656-90af-13aa9407465b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6fce} kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Stopping container ensemble-graph-f6fce metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3d8724b00b5 namespace: kserve-ci-e2e-test resourceVersion: "24233" uid: 71505e22-62eb-45dc-ae13-ae140acd665b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-f6fce} kind: Pod name: ensemble-graph-f6fce-77754d84cf-hl57v namespace: kserve-ci-e2e-test resourceVersion: "20698" uid: 4ed591bf-52a9-4670-a732-d500179252ed kind: Event lastTimestamp: "2026-04-22T19:16:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:55Z" name: ensemble-graph-f6fce-77754d84cf-hl57v.18a8c3d92629f661 namespace: kserve-ci-e2e-test resourceVersion: "24569" uid: 41c78322-1d06-42f1-925c-c6409fd94cac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-f6fce-77754d84cf namespace: kserve-ci-e2e-test resourceVersion: "20692" uid: 9d149600-002f-4126-b0e5-f96dfe4c03e7 kind: Event lastTimestamp: "2026-04-22T19:08:12Z" message: 'Created pod: ensemble-graph-f6fce-77754d84cf-hl57v' metadata: creationTimestamp: "2026-04-22T19:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:12Z" name: ensemble-graph-f6fce-77754d84cf.18a8c3654b731ee2 namespace: kserve-ci-e2e-test resourceVersion: "20697" uid: 1eea71b7-e6d7-4151-afa6-35152777aec2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20690" uid: e55a6456-c79e-4ced-b365-2db135dc45be kind: Event lastTimestamp: "2026-04-22T19:08:12Z" message: Scaled up replica set ensemble-graph-f6fce-77754d84cf from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:12Z" name: ensemble-graph-f6fce.18a8c3654ae9f4ca namespace: kserve-ci-e2e-test resourceVersion: "20693" uid: 617d38b3-40a7-43c9-8b14-ac0d332131f6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: 00db0f6d-b2b4-4d6b-98e1-f105441acd9b kind: Event lastTimestamp: "2026-04-22T19:08:12Z" message: InferenceGraph [ensemble-graph-f6fce] is Ready metadata: creationTimestamp: "2026-04-22T19:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:12Z" name: ensemble-graph-f6fce.18a8c3654fb134e2 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: 748126df-b172-4d1d-a4af-7d64eaf77246 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: c7a64d4f-c00e-44d7-95ba-01a1532a35ca kind: Event lastTimestamp: "2026-04-22T19:08:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:57Z" name: ensemble-graph-f6fce.18a8c368c95e7d62 namespace: kserve-ci-e2e-test resourceVersion: "21131" uid: a9a75d55-da7e-477b-9aac-0be654d02669 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:08:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20691" uid: c7a64d4f-c00e-44d7-95ba-01a1532a35ca kind: Event lastTimestamp: "2026-04-22T19:08:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:57Z" name: ensemble-graph-f6fce.18a8c368c9a15ed5 namespace: kserve-ci-e2e-test resourceVersion: "21132" uid: 5b383201-42b9-4976-ab16-b7319e9cbf10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:57:23.582981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0e291-predictor-748c9cfd47-j5dmj to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:57:23Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce21c0b346 namespace: kserve-ci-e2e-test resourceVersion: "15160" uid: f08210dd-0e3b-4a85-b2cb-a83713c2fbf0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0e291-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce32f389e9 namespace: kserve-ci-e2e-test resourceVersion: "15176" uid: 9b9a54ba-14e6-4c8c-b0ad-fc9ac7cab2a1 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15167" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce5efa1797 namespace: kserve-ci-e2e-test resourceVersion: "15193" uid: 4d71316c-7578-40f0-9b35-c1057cb94e1a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce603728f8 namespace: kserve-ci-e2e-test resourceVersion: "15196" uid: 7e8b774e-0470-493d-96ca-977983b178ef reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce63ac6cbf namespace: kserve-ci-e2e-test resourceVersion: "15198" uid: ef49033e-611c-482a-a260-060c4c17494f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce648a7d3f namespace: kserve-ci-e2e-test resourceVersion: "15199" uid: ea1e7c26-0f45-4262-8f6d-5711dd5d8b89 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce6495f8f0 namespace: kserve-ci-e2e-test resourceVersion: "15201" uid: 7c28d25d-c4bb-4e38-89f7-57322b118be8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce6775ff0d namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: cc5b9c3b-f40f-4e7e-b789-0dc1425b606d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2ce67e96020 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: bc14af7f-0946-4085-b464-0c1308c14a9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T18:58:02Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:58:02Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c2cebb62dd56 namespace: kserve-ci-e2e-test resourceVersion: "15586" uid: 695ed9de-2e1b-4904-a70a-199b6d3e438e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c34f53bdf7a4 namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: a501a105-1f54-4cb5-82af-9d6d9a2d918c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj namespace: kserve-ci-e2e-test resourceVersion: "15158" uid: d9c07e2c-f953-4b9e-8241-dbf9834c901f kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj.18a8c34f53be72ce namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: 7f885bc9-e1e6-4546-a7cd-2bbdd3d69e14 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0e291-predictor-748c9cfd47 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: 95da5918-ad21-4d7d-8ed0-d0a43031b664 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: 'Created pod: error-404-isvc-0e291-predictor-748c9cfd47-j5dmj' metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:23Z" name: error-404-isvc-0e291-predictor-748c9cfd47.18a8c2ce213bc5b8 namespace: kserve-ci-e2e-test resourceVersion: "15157" uid: b2a0f907-2c77-4901-880e-da174c898251 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15132" uid: 986c7109-68da-468c-90b8-da02cc11d3eb kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Scaled up replica set error-404-isvc-0e291-predictor-748c9cfd47 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:23Z" name: error-404-isvc-0e291-predictor.18a8c2ce1f5d7784 namespace: kserve-ci-e2e-test resourceVersion: "15143" uid: e7bfc9bc-2975-40b0-8930-881736e31db9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 0e5a26d3-8228-4d4e-8fd8-72c4e46be925 kind: Event lastTimestamp: "2026-04-22T18:57:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:53Z" name: error-404-isvc-0e291-predictor.18a8c2d19dc8a1ed namespace: kserve-ci-e2e-test resourceVersion: "15514" uid: 90e0a4b3-e276-422b-ad4f-586368d14e60 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15138" uid: 0e5a26d3-8228-4d4e-8fd8-72c4e46be925 kind: Event lastTimestamp: "2026-04-22T18:57:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:53Z" name: error-404-isvc-0e291-predictor.18a8c2d19e256ebb namespace: kserve-ci-e2e-test resourceVersion: "15515" uid: 09498cd6-b2e5-443e-9d0b-c0c484c49717 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 0e5a26d3-8228-4d4e-8fd8-72c4e46be925 kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: error-404-isvc-0e291-predictor.18a8c2d89b90820e namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 571daa8a-878d-4f79-b4b7-eaba7acc9ed6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15359" uid: 0e5a26d3-8228-4d4e-8fd8-72c4e46be925 kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: error-404-isvc-0e291-predictor.18a8c2d89bd415d3 namespace: kserve-ci-e2e-test resourceVersion: "15911" uid: cd3b0a80-1930-4522-919c-6be2bfe0291e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15109" uid: b77aca32-9227-4e14-b47f-d04e408eb7c0 kind: Event lastTimestamp: "2026-04-22T18:58:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:58:12Z" name: error-404-isvc-0e291.18a8c2ce1f2f9992 namespace: kserve-ci-e2e-test resourceVersion: "15677" uid: 15489193-b49b-4346-986f-7afadff4a4b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: b77aca32-9227-4e14-b47f-d04e408eb7c0 kind: Event lastTimestamp: "2026-04-22T18:58:12Z" message: InferenceService [error-404-isvc-0e291] is Ready metadata: creationTimestamp: "2026-04-22T18:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:58:12Z" name: error-404-isvc-0e291.18a8c2d9756dc293 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: cdecd769-d41a-4341-8fd0-1efcae6ffb38 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:06:38.519186Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19451" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-23058-predictor-7f696595cc-bzs96 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f56872eb1 namespace: kserve-ci-e2e-test resourceVersion: "19455" uid: f4dcf3b5-cef0-4f2e-b60f-b3a4904c4b0b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-23058-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f611b49cf namespace: kserve-ci-e2e-test resourceVersion: "19465" uid: e09996f8-8e52-44b8-8668-57249ebd9b61 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19460" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f944f7133 namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: a57cd39d-44d0-419f-ad8d-8286bfd0c0f0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f95afa4cb namespace: kserve-ci-e2e-test resourceVersion: "19493" uid: c2033601-68c1-40bf-a844-95b86de8e81d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f9884a283 namespace: kserve-ci-e2e-test resourceVersion: "19496" uid: b3e84768-0b5b-4bc0-b487-ab0796cdeb1a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f98f3a51c namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 5132d8a8-0e6f-4f11-924c-a42c8afba48a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f98ff97c4 namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: 16fb38bd-dd4b-4c89-aacd-76b5060b69fd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f9c7890e1 namespace: kserve-ci-e2e-test resourceVersion: "19499" uid: 2b0fa02c-6ba2-4ab5-9664-71c66f68a04c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:06:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:39Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34f9ced9a9c namespace: kserve-ci-e2e-test resourceVersion: "19500" uid: c9e37145-aaab-43d8-ba89-86e58726d683 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:07:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:06:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:17Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c34fee8d3aef namespace: kserve-ci-e2e-test resourceVersion: "20010" uid: 5d0ab61c-162f-4b09-af54-9abcd5950681 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c35faf1e3e40 namespace: kserve-ci-e2e-test resourceVersion: "20412" uid: 98f64b74-6093-43c7-b3cb-e9731f1ca7ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-23058-predictor-7f696595cc-bzs96 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 27f5dc48-f149-4b40-a291-41d34033c8d2 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-23058-predictor-7f696595cc-bzs96.18a8c35faf1ec3a5 namespace: kserve-ci-e2e-test resourceVersion: "20416" uid: 9cb822d7-afde-4eca-b1bd-c929e2656adf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-23058-predictor-7f696595cc namespace: kserve-ci-e2e-test resourceVersion: "19444" uid: aa2aa71f-52dc-4592-8aac-61ce198c6ab2 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Created pod: error-404-isvc-23058-predictor-7f696595cc-bzs96' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058-predictor-7f696595cc.18a8c34f5624d174 namespace: kserve-ci-e2e-test resourceVersion: "19452" uid: e9492488-4e2f-4b39-a801-4685a046da29 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19440" uid: e7273252-1baf-4a26-b850-a850b55922c3 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Scaled up replica set error-404-isvc-23058-predictor-7f696595cc from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058-predictor.18a8c34f54adaaaa namespace: kserve-ci-e2e-test resourceVersion: "19446" uid: 45c440fb-0aa5-488d-84ac-6b0dd0aa1fda reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 0a502524-48ce-41e2-91c8-1fc9cf7ee8d2 kind: Event lastTimestamp: "2026-04-22T19:07:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:08Z" name: error-404-isvc-23058-predictor.18a8c352d31abf1d namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: 83a0dddd-06be-46f8-9c45-524ba84c0414 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 0a502524-48ce-41e2-91c8-1fc9cf7ee8d2 kind: Event lastTimestamp: "2026-04-22T19:07:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:08Z" name: error-404-isvc-23058-predictor.18a8c352d37a4cb1 namespace: kserve-ci-e2e-test resourceVersion: "19797" uid: 3d150fdd-3207-4b01-862c-0cec07640985 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 0a502524-48ce-41e2-91c8-1fc9cf7ee8d2 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: error-404-isvc-23058-predictor.18a8c359d0e23ee3 namespace: kserve-ci-e2e-test resourceVersion: "20231" uid: 0399302a-3d48-4ff6-96da-a60f15531845 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19637" uid: 0a502524-48ce-41e2-91c8-1fc9cf7ee8d2 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: error-404-isvc-23058-predictor.18a8c359d13f0c51 namespace: kserve-ci-e2e-test resourceVersion: "20232" uid: 21a885d7-50e0-4d0f-97ad-e16586b122f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: c09cac98-3a9f-4e43-aed8-65750ee3abae kind: Event lastTimestamp: "2026-04-22T19:07:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:27Z" name: error-404-isvc-23058.18a8c34f54831bbd namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 233933e1-23a5-4073-b63e-347641c65812 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: c09cac98-3a9f-4e43-aed8-65750ee3abae kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Failed to update status for InferenceService "error-404-isvc-23058": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-23058": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058.18a8c34f54da4554 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: 13a8fe3c-5d9d-43f6-9c13-631cd53f4336 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19410" uid: c09cac98-3a9f-4e43-aed8-65750ee3abae kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-23058": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:38Z" name: error-404-isvc-23058.18a8c34f54da9744 namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: 508b7a5a-4958-4ee4-ba40-96af127850da reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "20113" uid: c09cac98-3a9f-4e43-aed8-65750ee3abae kind: Event lastTimestamp: "2026-04-22T19:07:27Z" message: InferenceService [error-404-isvc-23058] is Ready metadata: creationTimestamp: "2026-04-22T19:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:27Z" name: error-404-isvc-23058.18a8c35aa8ab165b namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: ee0af63b-bfe5-41da-b1c8-207df0f0f3dd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:17:47.943725Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25280" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-352f4-predictor-6f5f47949-9nfnv to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb33570e4d namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: c740bb1a-051c-4c27-8177-02f18514bb9b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25289" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb4c802605 namespace: kserve-ci-e2e-test resourceVersion: "25300" uid: c878eda2-f511-460e-96a2-2695e8847ff8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb4dbf4147 namespace: kserve-ci-e2e-test resourceVersion: "25302" uid: 0794ae62-aadd-4e32-8a35-7f3f288951d8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb50b677c1 namespace: kserve-ci-e2e-test resourceVersion: "25304" uid: 131e9614-3582-463b-b589-263343eb4f64 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb5125e9d7 namespace: kserve-ci-e2e-test resourceVersion: "25307" uid: 24820791-d013-45e7-847c-1a9d3f7f4282 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb5131c8e8 namespace: kserve-ci-e2e-test resourceVersion: "25309" uid: 284bfe2a-81be-4969-be86-13037d28c7b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb547f2fe8 namespace: kserve-ci-e2e-test resourceVersion: "25310" uid: b5af4995-51ed-4021-801c-1677504486cf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3eb54f65440 namespace: kserve-ci-e2e-test resourceVersion: "25311" uid: 8246aac8-176e-493b-ba8f-13a904423d30 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:18:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:26Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c3ebc64c8c3d namespace: kserve-ci-e2e-test resourceVersion: "25698" uid: 1c1e5f66-9e89-40e1-b0dd-565e51329b14 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:27:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:02Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c46c61a79b7b namespace: kserve-ci-e2e-test resourceVersion: "29665" uid: c504b7a8-b895-4b8d-a8f6-189debca86d4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv namespace: kserve-ci-e2e-test resourceVersion: "25282" uid: 8317a774-5ed9-452a-8edc-6a717007cb36 kind: Event lastTimestamp: "2026-04-22T19:27:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:02Z" name: error-404-isvc-352f4-predictor-6f5f47949-9nfnv.18a8c46c61a85bc8 namespace: kserve-ci-e2e-test resourceVersion: "29666" uid: 253e1760-af9f-4f8f-90e3-1706a5bd4f31 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-352f4-predictor-6f5f47949 namespace: kserve-ci-e2e-test resourceVersion: "25271" uid: 5050373f-56d6-4069-b666-9efaefba6ba0 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'Created pod: error-404-isvc-352f4-predictor-6f5f47949-9nfnv' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-352f4-predictor-6f5f47949.18a8c3eb32e4612b namespace: kserve-ci-e2e-test resourceVersion: "25281" uid: 1ea1c8b0-c6c0-4c0b-b75b-f414a5267314 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: 93eb704f-6d6c-4416-8eef-94fa929cc078 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Scaled up replica set error-404-isvc-352f4-predictor-6f5f47949 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-352f4-predictor.18a8c3eb316c2de4 namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: 6507a446-5c9b-4267-8a76-0040d4d645c9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 70ad0f6f-b93e-4412-8e83-5ef58c6869df kind: Event lastTimestamp: "2026-04-22T19:18:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:17Z" name: error-404-isvc-352f4-predictor.18a8c3eeb04d421f namespace: kserve-ci-e2e-test resourceVersion: "25628" uid: 6d4d529f-30bf-48b2-8587-310cf40ce238 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 70ad0f6f-b93e-4412-8e83-5ef58c6869df kind: Event lastTimestamp: "2026-04-22T19:18:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:17Z" name: error-404-isvc-352f4-predictor.18a8c3eeb0b5be64 namespace: kserve-ci-e2e-test resourceVersion: "25629" uid: 6e230c39-5ba1-4bd2-98cd-36b58cc8b7c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 70ad0f6f-b93e-4412-8e83-5ef58c6869df kind: Event lastTimestamp: "2026-04-22T19:18:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:47Z" name: error-404-isvc-352f4-predictor.18a8c3f5ae2fb585 namespace: kserve-ci-e2e-test resourceVersion: "25907" uid: 39e0ead2-46bf-4d70-9e19-8fcd5e69e0db reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25469" uid: 70ad0f6f-b93e-4412-8e83-5ef58c6869df kind: Event lastTimestamp: "2026-04-22T19:18:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:47Z" name: error-404-isvc-352f4-predictor.18a8c3f5ae91feaf namespace: kserve-ci-e2e-test resourceVersion: "25910" uid: 1eb1bb83-4b50-43bb-8eb4-e8f293ab14fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: 6b8c46bf-b9a7-41a3-a34a-54aa56c74c03 kind: Event lastTimestamp: "2026-04-22T19:18:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:36Z" name: error-404-isvc-352f4.18a8c3eb3148e1d0 namespace: kserve-ci-e2e-test resourceVersion: "25796" uid: 5ca69158-7d1f-4fa9-afcf-08b453232c43 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: 6b8c46bf-b9a7-41a3-a34a-54aa56c74c03 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-352f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-352f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-352f4.18a8c3eb31919cde namespace: kserve-ci-e2e-test resourceVersion: "25276" uid: 516a5d7a-ca0e-41ef-8d35-ebd8dc69b7cb reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25245" uid: 6b8c46bf-b9a7-41a3-a34a-54aa56c74c03 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-352f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-352f4.18a8c3eb3191e90b namespace: kserve-ci-e2e-test resourceVersion: "25278" uid: 60d4fe51-04a3-4754-bb3c-04a73d45028b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25794" uid: 6b8c46bf-b9a7-41a3-a34a-54aa56c74c03 kind: Event lastTimestamp: "2026-04-22T19:18:36Z" message: InferenceService [error-404-isvc-352f4] is Ready metadata: creationTimestamp: "2026-04-22T19:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:36Z" name: error-404-isvc-352f4.18a8c3f681e199a2 namespace: kserve-ci-e2e-test resourceVersion: "25795" uid: a655ee2a-7f06-430a-a55d-ea2f5ed22fe6 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:06:02.598467Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19034" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c346f97c8fb3 namespace: kserve-ci-e2e-test resourceVersion: "19040" uid: 88b5a6d1-243b-416d-9b09-d9931ed49c31 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-50a5e-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3470461fcf5 namespace: kserve-ci-e2e-test resourceVersion: "19048" uid: e4c736eb-292c-44e4-8a72-33ba34882574 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19046" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c34736cd9bba namespace: kserve-ci-e2e-test resourceVersion: "19073" uid: e687cf8c-ef95-4745-9d67-ec3b8a0fc5e9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c347380d80d8 namespace: kserve-ci-e2e-test resourceVersion: "19076" uid: fc4c8d3b-e2fc-4998-9a6f-fe368e2fb953 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3473b2ff0d3 namespace: kserve-ci-e2e-test resourceVersion: "19077" uid: 743e9a34-f1ec-4ab2-8b83-9bf7c7aedb60 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3473ba19f68 namespace: kserve-ci-e2e-test resourceVersion: "19078" uid: b15e42cd-1384-4e07-b29e-941431bdeb2a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3473bae83b8 namespace: kserve-ci-e2e-test resourceVersion: "19079" uid: 198c96fd-f53d-4f6a-86cb-28dbec790b2a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3473eb60322 namespace: kserve-ci-e2e-test resourceVersion: "19080" uid: e689e3a9-7c2d-49d2-807c-15f1c074d4d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3473f2540e2 namespace: kserve-ci-e2e-test resourceVersion: "19081" uid: 56d72fa7-4e0c-47b4-9a02-c6c821a8327f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:06:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:40Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c347854e563f namespace: kserve-ci-e2e-test resourceVersion: "19511" uid: a9af0684-9011-45a7-873e-36e9794614ae reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c3575263e12a namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: f4d9a0fb-04f0-4439-b40f-da21ee3c1426 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c357526460ca namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: e69d2e74-3063-41d2-999a-fe834ce8cb93 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4 namespace: kserve-ci-e2e-test resourceVersion: "19037" uid: a321ae25-0e38-4211-b254-595a1170bd90 kind: Event lastTimestamp: "2026-04-22T19:07:16Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": context deadline exceeded' metadata: creationTimestamp: "2026-04-22T19:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:16Z" name: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4.18a8c35849138c08 namespace: kserve-ci-e2e-test resourceVersion: "20008" uid: 05ca02bb-cb2a-4324-b73f-545e0519bb8f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-50a5e-predictor-9c6d696b5 namespace: kserve-ci-e2e-test resourceVersion: "19026" uid: 9f48486f-35f4-46ae-9c34-5deae586583e kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'Created pod: error-404-isvc-50a5e-predictor-9c6d696b5-r8mz4' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e-predictor-9c6d696b5.18a8c346f8f5dce6 namespace: kserve-ci-e2e-test resourceVersion: "19036" uid: 7cffc438-d36a-43df-b0fe-48bd5a6a13d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19023" uid: 764f7d7b-bcea-4d73-b090-f6b0d8d5d2b8 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Scaled up replica set error-404-isvc-50a5e-predictor-9c6d696b5 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e-predictor.18a8c346f791374d namespace: kserve-ci-e2e-test resourceVersion: "19029" uid: cc001d70-f382-4446-b55c-a6bc9425f1e3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19025" uid: 4b79a0f4-9c77-4eac-8396-1b119fb828a2 kind: Event lastTimestamp: "2026-04-22T19:06:47Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:47Z" name: error-404-isvc-50a5e-predictor.18a8c34a75f91681 namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: 9ffdc3dc-aea6-4135-8292-f8d772fa4466 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19025" uid: 4b79a0f4-9c77-4eac-8396-1b119fb828a2 kind: Event lastTimestamp: "2026-04-22T19:06:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:47Z" name: error-404-isvc-50a5e-predictor.18a8c34a7657ac13 namespace: kserve-ci-e2e-test resourceVersion: "19575" uid: e3b0ed6d-a0db-482f-bc03-14111de1a482 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 4b79a0f4-9c77-4eac-8396-1b119fb828a2 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: error-404-isvc-50a5e-predictor.18a8c354f3347340 namespace: kserve-ci-e2e-test resourceVersion: "19731" uid: 524a79a7-57dc-4ffa-94b7-60d00848d4f5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19200" uid: 4b79a0f4-9c77-4eac-8396-1b119fb828a2 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: error-404-isvc-50a5e-predictor.18a8c354f3d4e09c namespace: kserve-ci-e2e-test resourceVersion: "19733" uid: ececeb67-61ed-47bc-b299-b874bffe5aaf reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 7c76bcc6-45f6-4095-ba20-206f9651774a kind: Event lastTimestamp: "2026-04-22T19:06:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:51Z" name: error-404-isvc-50a5e.18a8c346f76aa41d namespace: kserve-ci-e2e-test resourceVersion: "19614" uid: cb2f4725-fe7e-4f7c-b6a3-b48da8ac3f0f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 7c76bcc6-45f6-4095-ba20-206f9651774a kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'Failed to update status for InferenceService "error-404-isvc-50a5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-50a5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e.18a8c346f7b2ed82 namespace: kserve-ci-e2e-test resourceVersion: "19031" uid: 729800f9-bed3-431c-9d39-55f7e7fca65f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18993" uid: 7c76bcc6-45f6-4095-ba20-206f9651774a kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-50a5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-50a5e.18a8c346f7b350f7 namespace: kserve-ci-e2e-test resourceVersion: "19032" uid: 261c0418-1073-4ae4-b76b-16279aa3b6f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "19612" uid: 7c76bcc6-45f6-4095-ba20-206f9651774a kind: Event lastTimestamp: "2026-04-22T19:06:51Z" message: InferenceService [error-404-isvc-50a5e] is Ready metadata: creationTimestamp: "2026-04-22T19:06:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:51Z" name: error-404-isvc-50a5e.18a8c3523f55bc0b namespace: kserve-ci-e2e-test resourceVersion: "19613" uid: c0f9cf35-546e-4007-b9f5-ad63541f973b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:56:47.676734Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-70f3d-predictor-568f669b8f-klg8v to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5c592cf3c namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 7f9e0a20-3b72-4bff-af28-adaffa5bfb26 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14666" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5deefb0a2 namespace: kserve-ci-e2e-test resourceVersion: "14677" uid: f4150e91-7f3f-4aad-b9e1-02bc5faabfa7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e04703d3 namespace: kserve-ci-e2e-test resourceVersion: "14679" uid: ded51356-4baf-47fa-a1f4-9b19f5649e9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e318ec80 namespace: kserve-ci-e2e-test resourceVersion: "14680" uid: 423647d6-6413-41bb-bb69-3b8e93b0c71d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e3893a40 namespace: kserve-ci-e2e-test resourceVersion: "14681" uid: 6f9f5cfc-1fff-4be8-9536-18bd59f4e0e6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e393d05d namespace: kserve-ci-e2e-test resourceVersion: "14683" uid: e882d3f4-9058-4b4d-93ef-75c3af10c21d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e65b6f1c namespace: kserve-ci-e2e-test resourceVersion: "14685" uid: 19b62c60-d991-4f9c-94b8-d0407d05887a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c5e6c9e2f5 namespace: kserve-ci-e2e-test resourceVersion: "14686" uid: 43b8734c-4e7e-4a00-bf08-a9aca90b9674 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:56:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T18:57:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:56:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:26Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c2c6513eeeeb namespace: kserve-ci-e2e-test resourceVersion: "15225" uid: 511971e4-2536-442a-b500-afed48267152 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c346f6ea6628 namespace: kserve-ci-e2e-test resourceVersion: "19024" uid: 9860e67b-9d22-4c14-8323-23fc9b6511e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 4b913f2e-9481-4e91-9946-9d957f29ae31 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: error-404-isvc-70f3d-predictor-568f669b8f-klg8v.18a8c346f6eafe62 namespace: kserve-ci-e2e-test resourceVersion: "19027" uid: ac6a3d70-58e3-42ad-ac8c-21e5a178422c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-70f3d-predictor-568f669b8f namespace: kserve-ci-e2e-test resourceVersion: "14650" uid: 55dd2cab-01f6-4bc2-acd6-f65216905d2e kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'Created pod: error-404-isvc-70f3d-predictor-568f669b8f-klg8v' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-70f3d-predictor-568f669b8f.18a8c2c5c5312a19 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 5ab7eccb-81e4-4672-a4a3-221da20720ab reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14648" uid: 08c16550-4f2f-4bdc-8817-be58cfd9cdd3 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Scaled up replica set error-404-isvc-70f3d-predictor-568f669b8f from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-70f3d-predictor.18a8c2c5c3c94a30 namespace: kserve-ci-e2e-test resourceVersion: "14652" uid: f943616c-dcf0-4d66-a94c-07b8657ed9a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 5d04819e-ddaf-4fda-97e4-6533ccb6a221 kind: Event lastTimestamp: "2026-04-22T18:57:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:02Z" name: error-404-isvc-70f3d-predictor.18a8c2c9423bac6d namespace: kserve-ci-e2e-test resourceVersion: "14839" uid: 50781c52-e522-439a-a69a-dc0084ff5cf0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14649" uid: 5d04819e-ddaf-4fda-97e4-6533ccb6a221 kind: Event lastTimestamp: "2026-04-22T18:57:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:02Z" name: error-404-isvc-70f3d-predictor.18a8c2c942968e0e namespace: kserve-ci-e2e-test resourceVersion: "14841" uid: 48e340bb-e097-4180-9d5c-c69a334a2410 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 5d04819e-ddaf-4fda-97e4-6533ccb6a221 kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: error-404-isvc-70f3d-predictor.18a8c2ccc138ad5f namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 091d7bba-7bd3-47eb-a3d8-e18cb47e2e64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:57:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14840" uid: 5d04819e-ddaf-4fda-97e4-6533ccb6a221 kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:57:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: error-404-isvc-70f3d-predictor.18a8c2ccc199ad46 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 432861d8-7d99-49db-80ab-bb2d8faa8946 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 77c53719-cd87-4cf1-af18-5521163cf7eb kind: Event lastTimestamp: "2026-04-22T18:57:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:36Z" name: error-404-isvc-70f3d.18a8c2c5c3a051ae namespace: kserve-ci-e2e-test resourceVersion: "15337" uid: 7c03702c-35fd-4ebc-94ed-291413bc9011 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 77c53719-cd87-4cf1-af18-5521163cf7eb kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'Failed to update status for InferenceService "error-404-isvc-70f3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-70f3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-70f3d.18a8c2c5c4080b39 namespace: kserve-ci-e2e-test resourceVersion: "14654" uid: 1152efa3-e13e-45c6-8806-838410234129 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14616" uid: 77c53719-cd87-4cf1-af18-5521163cf7eb kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-70f3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-70f3d.18a8c2c5c4086f06 namespace: kserve-ci-e2e-test resourceVersion: "14655" uid: bf56336a-6e8c-4c51-8362-bed57dc1950c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15335" uid: 77c53719-cd87-4cf1-af18-5521163cf7eb kind: Event lastTimestamp: "2026-04-22T18:57:36Z" message: InferenceService [error-404-isvc-70f3d] is Ready metadata: creationTimestamp: "2026-04-22T18:57:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:36Z" name: error-404-isvc-70f3d.18a8c2d10bb3fa9e namespace: kserve-ci-e2e-test resourceVersion: "15336" uid: cd736198-fe5e-42eb-af1f-6377189db1e5 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:55:13.509925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13544" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-9844b-predictor-57bf65897f-5ns59 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:55:13Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2afd8cb5d52 namespace: kserve-ci-e2e-test resourceVersion: "13552" uid: 2e80287e-3898-4514-915e-9f1e830dfe19 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-9844b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:13Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2afe9c79501 namespace: kserve-ci-e2e-test resourceVersion: "13572" uid: 526574e2-aec2-4d16-b7a6-b2710d8ac747 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13562" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:55:14Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b0167eb3ab namespace: kserve-ci-e2e-test resourceVersion: "13618" uid: f2d82a92-8d85-4820-b88f-aa446e7b1d41 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:14Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b017c960f3 namespace: kserve-ci-e2e-test resourceVersion: "13620" uid: 31584888-abbf-4068-9a36-37d1bfe2b862 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" in 13.47s (13.47s including waiting). Image size: 1334262375 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b33ab3f89a namespace: kserve-ci-e2e-test resourceVersion: "13769" uid: c1b837d0-aac3-4637-ba6f-78f268e94bae reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b33e82a98c namespace: kserve-ci-e2e-test resourceVersion: "13770" uid: 2bbdcb38-6abc-443e-b099-ace45fef9559 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b33ef78cac namespace: kserve-ci-e2e-test resourceVersion: "13771" uid: 7b00b54c-6cba-4059-9581-d6536af87ede reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b33f03885b namespace: kserve-ci-e2e-test resourceVersion: "13772" uid: dd846619-4d05-4546-8cc0-2ffa0171aebd reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:30Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.244s (2.244s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:30Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b3c4c548c1 namespace: kserve-ci-e2e-test resourceVersion: "13813" uid: 76344548-d786-4620-ad20-1b662ce594e1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:30Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b3c8261dd4 namespace: kserve-ci-e2e-test resourceVersion: "13814" uid: f537a794-eb8f-453e-afad-e68459c75e1d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:55:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:30Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b3c89b5a90 namespace: kserve-ci-e2e-test resourceVersion: "13816" uid: 06c47e3e-2666-420a-8c4c-1dd1f96167eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:56:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:06Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2b3db9d2aa6 namespace: kserve-ci-e2e-test resourceVersion: "14154" uid: 522863f5-0571-4e63-aa31-36d1740d1350 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2c5c211ae52 namespace: kserve-ci-e2e-test resourceVersion: "14641" uid: 59ab5255-d037-4f14-9f81-b53355241784 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-9844b-predictor-57bf65897f-5ns59 namespace: kserve-ci-e2e-test resourceVersion: "13547" uid: eba18b2a-b04d-41df-a35c-7aded0b0ca0d kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: error-404-isvc-9844b-predictor-57bf65897f-5ns59.18a8c2c5c2129abc namespace: kserve-ci-e2e-test resourceVersion: "14645" uid: cc2b8a80-746e-4d37-a014-b1b30a45c73e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-9844b-predictor-57bf65897f namespace: kserve-ci-e2e-test resourceVersion: "13529" uid: 1af7ebfc-18f4-4315-a25a-701fef02fa03 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'Created pod: error-404-isvc-9844b-predictor-57bf65897f-5ns59' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: error-404-isvc-9844b-predictor-57bf65897f.18a8c2afd8632cba namespace: kserve-ci-e2e-test resourceVersion: "13546" uid: 92f986f8-5820-495b-99d6-96b83476fcec reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13527" uid: 0d242b54-3401-486b-9ef1-ea9366f718b6 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Scaled up replica set error-404-isvc-9844b-predictor-57bf65897f from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: error-404-isvc-9844b-predictor.18a8c2afd4c0d38a namespace: kserve-ci-e2e-test resourceVersion: "13531" uid: f223023a-8500-41e5-8a73-42aed2982db0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13530" uid: fe132450-360c-4215-8561-b1aa9f5d6ee5 kind: Event lastTimestamp: "2026-04-22T18:56:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:13Z" name: error-404-isvc-9844b-predictor.18a8c2b3537ec3c4 namespace: kserve-ci-e2e-test resourceVersion: "14207" uid: 644b5949-4d32-499c-8ac3-0da95265db6a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13530" uid: fe132450-360c-4215-8561-b1aa9f5d6ee5 kind: Event lastTimestamp: "2026-04-22T18:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:13Z" name: error-404-isvc-9844b-predictor.18a8c2b353d9a5d3 namespace: kserve-ci-e2e-test resourceVersion: "14208" uid: 771f3771-607e-4072-a3fe-6a7e0ca668f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: fe132450-360c-4215-8561-b1aa9f5d6ee5 kind: Event lastTimestamp: "2026-04-22T18:56:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:43Z" name: error-404-isvc-9844b-predictor.18a8c2c14e9009f4 namespace: kserve-ci-e2e-test resourceVersion: "14505" uid: 2b72fe6a-66b9-420e-aa02-1cc0e1f42a72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13782" uid: fe132450-360c-4215-8561-b1aa9f5d6ee5 kind: Event lastTimestamp: "2026-04-22T18:56:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:43Z" name: error-404-isvc-9844b-predictor.18a8c2c14ef2c970 namespace: kserve-ci-e2e-test resourceVersion: "14506" uid: fb720633-39dc-42e7-af05-43a572376741 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "13512" uid: cac067ca-3250-4e4d-881a-d6cd97639097 kind: Event lastTimestamp: "2026-04-22T18:56:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:16Z" name: error-404-isvc-9844b.18a8c2afd4f13479 namespace: kserve-ci-e2e-test resourceVersion: "14260" uid: ac6827d9-c7ad-4e8e-a516-721ffe03bf6d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "14258" uid: cac067ca-3250-4e4d-881a-d6cd97639097 kind: Event lastTimestamp: "2026-04-22T18:56:16Z" message: InferenceService [error-404-isvc-9844b] is Ready metadata: creationTimestamp: "2026-04-22T18:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:16Z" name: error-404-isvc-9844b.18a8c2be96fcd04e namespace: kserve-ci-e2e-test resourceVersion: "14259" uid: 3c055a34-c31e-4f57-a5ad-0164288eedac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:16:27.628254Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24308" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d8802a30bf namespace: kserve-ci-e2e-test resourceVersion: "24313" uid: c9c671f5-97b9-4643-85f3-f55942780a48 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24319" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d899d94981 namespace: kserve-ci-e2e-test resourceVersion: "24329" uid: 16020bee-7128-479a-82e7-41c5e0652a66 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d89b40b0ed namespace: kserve-ci-e2e-test resourceVersion: "24331" uid: f2687cdd-a151-451c-ad3f-a34341533cce reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d89ed34fe1 namespace: kserve-ci-e2e-test resourceVersion: "24332" uid: f526a844-1674-43da-9b38-40c1b148a0e3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d89f5c5326 namespace: kserve-ci-e2e-test resourceVersion: "24333" uid: 5e45bf0a-ea05-4823-9ce4-a9e655b5916c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d89f697f0b namespace: kserve-ci-e2e-test resourceVersion: "24334" uid: 3ec00baa-257a-4b89-841c-8759a42f4509 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d8a392ff01 namespace: kserve-ci-e2e-test resourceVersion: "24336" uid: 9a827d37-7cd1-40c5-aee5-7bc187f34bf9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d8a409dfd4 namespace: kserve-ci-e2e-test resourceVersion: "24337" uid: 6748c729-7a2c-4996-8f0e-8bb97fba3271 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:17:06Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:06Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3d9134ea65a namespace: kserve-ci-e2e-test resourceVersion: "24787" uid: e13ddbe1-dad3-4db0-964d-6e4a710aefc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3eb316d93be namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: e4e443e3-b1fc-4bf8-9aab-f84397570603 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj namespace: kserve-ci-e2e-test resourceVersion: "24311" uid: 42e7e6aa-3ad6-42ee-a761-ddf4d8bf0141 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:47Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj.18a8c3eb316e2c0b namespace: kserve-ci-e2e-test resourceVersion: "25277" uid: 8e588f70-ae15-4170-990f-974fbb89197f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c8908-predictor-7c65ccdfd9 namespace: kserve-ci-e2e-test resourceVersion: "24300" uid: dad5e729-54ef-46fd-bb82-9a5a1621efc1 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'Created pod: error-404-isvc-c8908-predictor-7c65ccdfd9-5trtj' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-c8908-predictor-7c65ccdfd9.18a8c3d87fc6a6ad namespace: kserve-ci-e2e-test resourceVersion: "24310" uid: 8af313e4-f817-4eda-84a1-9a2f4a2ee832 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24296" uid: 007a4527-c7d8-4eff-b3d5-dd1f45979c37 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Scaled up replica set error-404-isvc-c8908-predictor-7c65ccdfd9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-c8908-predictor.18a8c3d87e495e8c namespace: kserve-ci-e2e-test resourceVersion: "24301" uid: 02a3fb48-475d-4f34-9383-3e198a9f35fd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: f9c6ffc5-8bab-495d-8a4b-e33e8faa65f9 kind: Event lastTimestamp: "2026-04-22T19:17:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:12Z" name: error-404-isvc-c8908-predictor.18a8c3dbfce4dcf9 namespace: kserve-ci-e2e-test resourceVersion: "24848" uid: d288d046-fe8b-44d9-ac44-79428b4957c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24299" uid: f9c6ffc5-8bab-495d-8a4b-e33e8faa65f9 kind: Event lastTimestamp: "2026-04-22T19:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:12Z" name: error-404-isvc-c8908-predictor.18a8c3dbfd41a6f8 namespace: kserve-ci-e2e-test resourceVersion: "24849" uid: eb231e37-f2b5-4edc-a0a5-836d57360fb5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24475" uid: f9c6ffc5-8bab-495d-8a4b-e33e8faa65f9 kind: Event lastTimestamp: "2026-04-22T19:17:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:42Z" name: error-404-isvc-c8908-predictor.18a8c3e67957aa4d namespace: kserve-ci-e2e-test resourceVersion: "25140" uid: bb3c8964-a93c-4c6b-83c2-f206f35564e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24475" uid: f9c6ffc5-8bab-495d-8a4b-e33e8faa65f9 kind: Event lastTimestamp: "2026-04-22T19:17:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:42Z" name: error-404-isvc-c8908-predictor.18a8c3e679ac7ef5 namespace: kserve-ci-e2e-test resourceVersion: "25141" uid: 83fb50b6-19db-443f-9a9e-6b5357a81e86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24269" uid: e524ab24-36af-479c-ac2a-84a1b5aa492f kind: Event lastTimestamp: "2026-04-22T19:17:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:16Z" name: error-404-isvc-c8908.18a8c3d87e4be4e7 namespace: kserve-ci-e2e-test resourceVersion: "24882" uid: bbd386ff-26c9-4cf6-92ec-2926d3614bda reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24269" uid: e524ab24-36af-479c-ac2a-84a1b5aa492f kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'Failed to update status for InferenceService "error-404-isvc-c8908": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c8908": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-c8908.18a8c3d87eb90247 namespace: kserve-ci-e2e-test resourceVersion: "24305" uid: f50793c7-e503-4196-abb0-7d2c62320c4b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24269" uid: e524ab24-36af-479c-ac2a-84a1b5aa492f kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c8908": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-c8908.18a8c3d87eb96351 namespace: kserve-ci-e2e-test resourceVersion: "24306" uid: 638777ac-4489-4b13-a912-cc2322b4291b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24880" uid: e524ab24-36af-479c-ac2a-84a1b5aa492f kind: Event lastTimestamp: "2026-04-22T19:17:16Z" message: InferenceService [error-404-isvc-c8908] is Ready metadata: creationTimestamp: "2026-04-22T19:17:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:16Z" name: error-404-isvc-c8908.18a8c3e3ce7bb9e0 namespace: kserve-ci-e2e-test resourceVersion: "24881" uid: 9af9881d-3e1f-4e43-9505-611eef0a8069 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:17:03.531450Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24729" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e0dc297b44 namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: 4e0e3d37-c971-468f-a64b-d94ec7b906e6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e7803-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e0e854727b namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 331f64b1-27ed-4064-a66a-99358bee3ac6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24738" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e11919b6db namespace: kserve-ci-e2e-test resourceVersion: "24764" uid: 2a2afb38-cb62-4a0b-8893-3ab1564217f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e11a6bfb21 namespace: kserve-ci-e2e-test resourceVersion: "24766" uid: 83233deb-d278-446c-9c12-5f7e8123fa07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e11dabe2de namespace: kserve-ci-e2e-test resourceVersion: "24768" uid: 90d5b330-441c-4a43-b56c-8166a6259958 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e11e1f74a4 namespace: kserve-ci-e2e-test resourceVersion: "24769" uid: 712524d6-c4e4-4c5a-8912-5e3f438150fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e11e2bcbb5 namespace: kserve-ci-e2e-test resourceVersion: "24770" uid: f9d7caee-b265-4125-baeb-deb396101a2f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e121d15351 namespace: kserve-ci-e2e-test resourceVersion: "24771" uid: 5bda22ee-fc7a-4932-b333-cdff31db46d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e1224bfd1e namespace: kserve-ci-e2e-test resourceVersion: "24772" uid: e3414f38-d6d8-4e0e-837b-71faa4e59cfe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:17:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:17:42Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:42Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c3e17d2aa05c namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: b2d6b53a-9087-4321-b65d-afb92f4756ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:34:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:33Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c4d53e24e6e2 namespace: kserve-ci-e2e-test resourceVersion: "32775" uid: 65680855-6276-4cc8-8c34-ce6cdf642ef2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl namespace: kserve-ci-e2e-test resourceVersion: "24731" uid: 55f046b3-38f6-4451-b8a1-cf7b0bdfc432 kind: Event lastTimestamp: "2026-04-22T19:34:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:33Z" name: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl.18a8c4d53e2587f2 namespace: kserve-ci-e2e-test resourceVersion: "32776" uid: 5bb6d3fa-9a45-43ab-99d9-1df0de19ce9c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e7803-predictor-6f9984fcb7 namespace: kserve-ci-e2e-test resourceVersion: "24721" uid: 16f2f473-be70-4b50-bfd6-f99b6dfe628e kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Created pod: error-404-isvc-e7803-predictor-6f9984fcb7-mv7xl' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803-predictor-6f9984fcb7.18a8c3e0dbc6d189 namespace: kserve-ci-e2e-test resourceVersion: "24730" uid: a39f48d8-3367-4583-a23a-bccddc8eda10 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24716" uid: 62a7bdc5-da1b-4c63-81c2-be6555ff670a kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Scaled up replica set error-404-isvc-e7803-predictor-6f9984fcb7 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803-predictor.18a8c3e0da5835de namespace: kserve-ci-e2e-test resourceVersion: "24723" uid: 201b2436-4213-4ff5-a736-e4ea9ece70c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: b9891642-3c87-4589-8339-7d4c6edb697b kind: Event lastTimestamp: "2026-04-22T19:18:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:18Z" name: error-404-isvc-e7803-predictor.18a8c3e458f37fbc namespace: kserve-ci-e2e-test resourceVersion: "25638" uid: 9de54ccd-af25-49c1-bc19-e75ee97d8054 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: b9891642-3c87-4589-8339-7d4c6edb697b kind: Event lastTimestamp: "2026-04-22T19:18:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:18Z" name: error-404-isvc-e7803-predictor.18a8c3e459500447 namespace: kserve-ci-e2e-test resourceVersion: "25639" uid: dcccd5e0-f769-40d1-b233-193407d7df84 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: b9891642-3c87-4589-8339-7d4c6edb697b kind: Event lastTimestamp: "2026-04-22T19:18:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:03Z" name: error-404-isvc-e7803-predictor.18a8c3eb56bb4597 namespace: kserve-ci-e2e-test resourceVersion: "25476" uid: 0905af97-6174-493d-a630-aefde35e59c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24908" uid: b9891642-3c87-4589-8339-7d4c6edb697b kind: Event lastTimestamp: "2026-04-22T19:18:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:03Z" name: error-404-isvc-e7803-predictor.18a8c3eb571d836d namespace: kserve-ci-e2e-test resourceVersion: "25477" uid: 89b6839a-22f9-46ce-91e1-85878b719e66 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: b200789f-0eb4-40dc-995a-9f6a58b1403e kind: Event lastTimestamp: "2026-04-22T19:17:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:52Z" name: error-404-isvc-e7803.18a8c3e0da59e64b namespace: kserve-ci-e2e-test resourceVersion: "25376" uid: 992e51b5-69bb-4e7b-aa68-ae01bfd8b2c5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: b200789f-0eb4-40dc-995a-9f6a58b1403e kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-e7803": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e7803": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803.18a8c3e0dacca9c8 namespace: kserve-ci-e2e-test resourceVersion: "24726" uid: a2349c56-a1b3-4235-96c9-89e5f78be29a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24687" uid: b200789f-0eb4-40dc-995a-9f6a58b1403e kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e7803": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-e7803.18a8c3e0daccf4b8 namespace: kserve-ci-e2e-test resourceVersion: "24727" uid: 66c077f8-5403-40f7-aff4-f197453fee40 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25374" uid: b200789f-0eb4-40dc-995a-9f6a58b1403e kind: Event lastTimestamp: "2026-04-22T19:17:52Z" message: InferenceService [error-404-isvc-e7803] is Ready metadata: creationTimestamp: "2026-04-22T19:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:52Z" name: error-404-isvc-e7803.18a8c3ec3740ad0c namespace: kserve-ci-e2e-test resourceVersion: "25375" uid: 0bb523ee-7d7b-42ca-9bc2-22f4e7fbe94f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:12.840638Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357543f0af7 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: 86c6d9c6-9a39-4024-8bcd-165f8cc9205a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f6fce-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c3575dab925b namespace: kserve-ci-e2e-test resourceVersion: "19945" uid: b01300e7-18c3-47c5-a9c9-e2e005b0da5d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19934" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357c7e136d9 namespace: kserve-ci-e2e-test resourceVersion: "19971" uid: 2148dd2b-f72b-41bb-9e48-6b65b4acc134 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357c9423e6f namespace: kserve-ci-e2e-test resourceVersion: "19973" uid: c3774b6e-0b52-466c-8ec1-2db20a638a02 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357ccf28514 namespace: kserve-ci-e2e-test resourceVersion: "19975" uid: f2524e6d-a6c8-4110-acbe-1e37a4c78b3a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357cd6580c8 namespace: kserve-ci-e2e-test resourceVersion: "19976" uid: 2fdc7eb9-fa5d-46d7-b7e2-db7a2d59eb2e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357cd71eedc namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 155cd055-a241-4351-b59c-0f433c60610c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357d04218c1 namespace: kserve-ci-e2e-test resourceVersion: "19978" uid: b3bd4bbe-1066-4592-af9e-6a46d5ddbad3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:14Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c357d0b577f0 namespace: kserve-ci-e2e-test resourceVersion: "19979" uid: f8232368-ad78-47d6-ae2b-cc546ae49677 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:07:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:52Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c3581c2fba35 namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: d2bf8310-6fd7-47f5-9477-7a087ecb0f9f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c3d87be678d7 namespace: kserve-ci-e2e-test resourceVersion: "24283" uid: 8b891679-4b87-40ec-b9fb-7d70b34bd220 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2 namespace: kserve-ci-e2e-test resourceVersion: "19928" uid: e2897854-0854-4157-8b02-eeaae615da73 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2.18a8c3d87be7580a namespace: kserve-ci-e2e-test resourceVersion: "24286" uid: a12ad397-ab88-42e0-934e-fac522b24356 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f6fce-predictor-98bc8cf4f namespace: kserve-ci-e2e-test resourceVersion: "19915" uid: c9c3b03a-1841-47f5-8e4c-1509dc65d71c kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'Created pod: error-404-isvc-f6fce-predictor-98bc8cf4f-nd4v2' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-f6fce-predictor-98bc8cf4f.18a8c35753d6ac65 namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: a25cc00c-ad1e-455d-ad1a-fb1e0e683f5f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19911" uid: 833a4998-8128-4f57-894f-9938cd4c3c1d kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Scaled up replica set error-404-isvc-f6fce-predictor-98bc8cf4f from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-f6fce-predictor.18a8c35752556806 namespace: kserve-ci-e2e-test resourceVersion: "19916" uid: 3c9e7033-097e-43ce-a1f5-263a83ba222f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: d2869fce-25f9-4efe-b6e7-68b8473b506d kind: Event lastTimestamp: "2026-04-22T19:08:27Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:27Z" name: error-404-isvc-f6fce-predictor.18a8c35ad103a4dd namespace: kserve-ci-e2e-test resourceVersion: "20848" uid: e8cb8311-4dde-4bd5-bd27-955a4970085c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19914" uid: d2869fce-25f9-4efe-b6e7-68b8473b506d kind: Event lastTimestamp: "2026-04-22T19:08:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:27Z" name: error-404-isvc-f6fce-predictor.18a8c35ad163d01b namespace: kserve-ci-e2e-test resourceVersion: "20849" uid: b7147241-e5e3-4d2a-82ea-c6cfec6e4a91 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: d2869fce-25f9-4efe-b6e7-68b8473b506d kind: Event lastTimestamp: "2026-04-22T19:07:57Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:57Z" name: error-404-isvc-f6fce-predictor.18a8c361cec1775e namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 59622913-da82-4dc5-8204-ecd3fc2fe2ee reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "20128" uid: d2869fce-25f9-4efe-b6e7-68b8473b506d kind: Event lastTimestamp: "2026-04-22T19:07:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:57Z" name: error-404-isvc-f6fce-predictor.18a8c361cf21077d namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: e12c825f-f224-4a8c-9075-be9ed8669c1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 449874a1-09c3-4827-b952-2d71de0d5588 kind: Event lastTimestamp: "2026-04-22T19:08:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:02Z" name: error-404-isvc-f6fce.18a8c357525564ca namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: 488cc4ab-6685-40f9-94ae-b9aa7b39cad2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 449874a1-09c3-4827-b952-2d71de0d5588 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-f6fce": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6fce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-f6fce.18a8c35752cf795b namespace: kserve-ci-e2e-test resourceVersion: "19921" uid: 48512030-b3bb-4e62-b336-acc0ccfd3305 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19884" uid: 449874a1-09c3-4827-b952-2d71de0d5588 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-f6fce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:12Z" name: error-404-isvc-f6fce.18a8c35752cfd435 namespace: kserve-ci-e2e-test resourceVersion: "19923" uid: 07d77035-6195-4684-994d-d41b28d6d601 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 449874a1-09c3-4827-b952-2d71de0d5588 kind: Event lastTimestamp: "2026-04-22T19:08:02Z" message: InferenceService [error-404-isvc-f6fce] is Ready metadata: creationTimestamp: "2026-04-22T19:08:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:02Z" name: error-404-isvc-f6fce.18a8c362d66e7d10 namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: 7f5112cc-487a-4bb8-ad3e-8dbb14dd9c9e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:48.763898Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20421" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35fb1706c37 namespace: kserve-ci-e2e-test resourceVersion: "20425" uid: 1e0126bb-573e-40ab-806e-9438fb57bc82 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ffb29-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35fb9b4b395 namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: 71b26ed2-1b86-43ca-889f-2ea65ccf99fa reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20429" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35fee7eae96 namespace: kserve-ci-e2e-test resourceVersion: "20454" uid: ad3bb1a0-073d-47ce-8898-e81879e72ff2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35fefe18d96 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: 867ef2cf-a6aa-4745-b8ae-e72d4074b130 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35ff33c8d09 namespace: kserve-ci-e2e-test resourceVersion: "20462" uid: 01b74e7f-c5ed-4fc8-8ad7-0668824012bc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35ff3b1e7fc namespace: kserve-ci-e2e-test resourceVersion: "20464" uid: 03906725-cf1a-4831-90f4-78a75dca9061 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35ff3bd76e5 namespace: kserve-ci-e2e-test resourceVersion: "20465" uid: 43608f40-8220-4ea1-b03f-4ec3fc6122eb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35ff6e84808 namespace: kserve-ci-e2e-test resourceVersion: "20466" uid: 864b4f13-36b3-4a6c-b56f-45f6bbe5e1f0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c35ff757262f namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 85145a3a-eeb5-4dae-b2f4-624c18eced41 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:08:27Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:27Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c3604a0f15dd namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 29665b36-7c53-41ec-85f8-48100dd69f20 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c3e0d99cea88 namespace: kserve-ci-e2e-test resourceVersion: "24717" uid: 64e82a03-e484-4fff-a958-bf8a78d1436e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6 namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: 2c07bb2c-1a22-4a32-a847-183c5af71d35 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6.18a8c3e0d99d8fa7 namespace: kserve-ci-e2e-test resourceVersion: "24722" uid: 6cdb4c1b-ff3c-4ba4-a1a8-8752fc1287e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ffb29-predictor-564df6b8f9 namespace: kserve-ci-e2e-test resourceVersion: "20413" uid: 2c7ddb2c-cb46-4cfb-aae1-b297b881ed9d kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'Created pod: error-404-isvc-ffb29-predictor-564df6b8f9-fssm6' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29-predictor-564df6b8f9.18a8c35fb0ed5f91 namespace: kserve-ci-e2e-test resourceVersion: "20422" uid: bd19c834-4931-4482-8dbc-5c516a8b97e3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20410" uid: d12cc508-59ab-4d80-847e-7d7460315365 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Scaled up replica set error-404-isvc-ffb29-predictor-564df6b8f9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29-predictor.18a8c35faf7e202b namespace: kserve-ci-e2e-test resourceVersion: "20415" uid: d872576c-4669-44f2-8652-548198796d35 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 87f1c99f-0b62-4982-a5fb-7852066857d5 kind: Event lastTimestamp: "2026-04-22T19:08:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:18Z" name: error-404-isvc-ffb29-predictor.18a8c3632df34263 namespace: kserve-ci-e2e-test resourceVersion: "20772" uid: d26656f3-0d43-4371-bc1d-7145d34d5274 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20411" uid: 87f1c99f-0b62-4982-a5fb-7852066857d5 kind: Event lastTimestamp: "2026-04-22T19:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:18Z" name: error-404-isvc-ffb29-predictor.18a8c3632e4f7227 namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: f5360fed-648e-4c14-b53b-420261b83547 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 87f1c99f-0b62-4982-a5fb-7852066857d5 kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:48Z" name: error-404-isvc-ffb29-predictor.18a8c36a2ba7bc68 namespace: kserve-ci-e2e-test resourceVersion: "21044" uid: f805e228-dbc6-4a3e-8a57-c8bdf38418e3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20613" uid: 87f1c99f-0b62-4982-a5fb-7852066857d5 kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:48Z" name: error-404-isvc-ffb29-predictor.18a8c36a2c03c18c namespace: kserve-ci-e2e-test resourceVersion: "21046" uid: 2fcddc20-0a75-4063-ac80-5d0a1a5a314f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 82f4f659-0b7f-45b4-bfe6-2a6ba9bf9ddf kind: Event lastTimestamp: "2026-04-22T19:08:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:37Z" name: error-404-isvc-ffb29.18a8c35faf556584 namespace: kserve-ci-e2e-test resourceVersion: "20926" uid: f1e6f0f5-f3c0-4307-8e2b-ee63cdf4e48f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 82f4f659-0b7f-45b4-bfe6-2a6ba9bf9ddf kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-ffb29": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ffb29": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29.18a8c35faf9b5fb9 namespace: kserve-ci-e2e-test resourceVersion: "20418" uid: 48ae4679-edb2-4fe1-b040-fb7d46e3e471 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: 82f4f659-0b7f-45b4-bfe6-2a6ba9bf9ddf kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ffb29": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:48Z" name: error-404-isvc-ffb29.18a8c35faf9bb611 namespace: kserve-ci-e2e-test resourceVersion: "20419" uid: b492a3c7-4b7e-49d2-8f04-834d2e9fd8f7 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20924" uid: 82f4f659-0b7f-45b4-bfe6-2a6ba9bf9ddf kind: Event lastTimestamp: "2026-04-22T19:08:37Z" message: InferenceService [error-404-isvc-ffb29] is Ready metadata: creationTimestamp: "2026-04-22T19:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:37Z" name: error-404-isvc-ffb29.18a8c36b042c0e06 namespace: kserve-ci-e2e-test resourceVersion: "20925" uid: ac85a1a6-3725-44ff-8c61-eab011188591 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:55:13.419698Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13515" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:55:13Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2afd36aa00a namespace: kserve-ci-e2e-test resourceVersion: "13521" uid: eb884119-4a79-48f6-8983-6d4603c024da reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13532" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:55:13Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2afed663676 namespace: kserve-ci-e2e-test resourceVersion: "13573" uid: ade65524-43de-44e7-b63a-015f83365ff3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:13Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2afeecd52cf namespace: kserve-ci-e2e-test resourceVersion: "13580" uid: 803ecb6c-795c-46da-a77b-5abe6dfe065c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:27Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 14.066s (14.066s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:27Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b3353967ff namespace: kserve-ci-e2e-test resourceVersion: "13764" uid: 9f689f98-0295-42c6-ab3f-77a7ff2a1678 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:27Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:27Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T18:55:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:27Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b3382d36ce namespace: kserve-ci-e2e-test resourceVersion: "13766" uid: d8296930-dd6c-425b-9eab-409bd6872474 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b338a5c304 namespace: kserve-ci-e2e-test resourceVersion: "13767" uid: eda51997-511e-47e1-a66f-3343650c7db2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:32Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-22T18:55:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:32Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b4534a13c6 namespace: kserve-ci-e2e-test resourceVersion: "13843" uid: f0ac3efb-2e47-4d69-8ef0-034b0a84eda3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 6.391s (6.391s including waiting). Image size: 1562510257 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d039f69d namespace: kserve-ci-e2e-test resourceVersion: "13907" uid: db171f2e-cfb6-44d0-8d74-65c15c6510ee reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d39b958b namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 26a50257-0909-4229-b82c-d1ab72187ddb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d415def9 namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: ee4c8499-8fd1-4610-877c-06362fdbc2ff reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d4354551 namespace: kserve-ci-e2e-test resourceVersion: "13910" uid: c1be4580-26d5-4ffc-9548-9719f6d8c817 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d734daa7 namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: d90b6add-dccd-4ac1-b444-7f5870e74f4a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:55:39Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:55:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:39Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b5d7a830e5 namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: b60899d6-9024-439a-8e37-3d97d156604d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T18:55:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:57:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:55:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:26Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2b63207202b namespace: kserve-ci-e2e-test resourceVersion: "15231" uid: d7168885-6432-4374-8d29-2064d2790c16 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2ce1cf0591c namespace: kserve-ci-e2e-test resourceVersion: "15121" uid: 3a84dbcf-d35b-4192-b2ff-1411cfafcce5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2ce1cf19d09 namespace: kserve-ci-e2e-test resourceVersion: "15125" uid: 24615aa4-9f12-474b-9e32-63f4fcf3261f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558 namespace: kserve-ci-e2e-test resourceVersion: "13518" uid: f9ca8ccb-2c94-4c3f-9d22-4cfaed4bde53 kind: Event lastTimestamp: "2026-04-22T18:57:26Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:26Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558.18a8c2cee03e3aff namespace: kserve-ci-e2e-test resourceVersion: "15230" uid: 1473ae7c-5ebe-467d-924f-8d889404c2b9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5b497dcd98 namespace: kserve-ci-e2e-test resourceVersion: "13506" uid: 7dc7341a-4f7b-4eb4-a248-1109338658a5 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5b497dcd98-8q558' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: isvc-sklearn-graph-1-predictor-5b497dcd98.18a8c2afd306eeb7 namespace: kserve-ci-e2e-test resourceVersion: "13517" uid: 7b9b3513-963f-4098-8b41-ad8c68f4672f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13504" uid: 763b8cba-b04f-473f-b283-b50169b9fc5e kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5b497dcd98 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: isvc-sklearn-graph-1-predictor.18a8c2afd013308f namespace: kserve-ci-e2e-test resourceVersion: "13508" uid: db7cb11a-ba48-4458-9113-c16be6bc6c09 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: a7a11e87-d2be-4867-9f4a-42abf07dfe88 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: isvc-sklearn-graph-1-predictor.18a8c2b34e8590f8 namespace: kserve-ci-e2e-test resourceVersion: "14944" uid: 110a4b41-9cc7-4b14-8fc4-448e616ff397 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13505" uid: a7a11e87-d2be-4867-9f4a-42abf07dfe88 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: isvc-sklearn-graph-1-predictor.18a8c2b34ee53161 namespace: kserve-ci-e2e-test resourceVersion: "14946" uid: 92ac4f99-88d3-407f-89f9-08027c5265b2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13778" uid: a7a11e87-d2be-4867-9f4a-42abf07dfe88 kind: Event lastTimestamp: "2026-04-22T18:56:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:58Z" name: isvc-sklearn-graph-1-predictor.18a8c2c14a854fbe namespace: kserve-ci-e2e-test resourceVersion: "14799" uid: 45075d81-6b87-47b7-8776-e36db2f82f8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:56:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13778" uid: a7a11e87-d2be-4867-9f4a-42abf07dfe88 kind: Event lastTimestamp: "2026-04-22T18:56:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:58Z" name: isvc-sklearn-graph-1-predictor.18a8c2c14ae70099 namespace: kserve-ci-e2e-test resourceVersion: "14800" uid: ad5dc125-d241-444a-bd50-d4a8b2588f1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13483" uid: 9dd9cd42-ae31-4282-8980-4f8073e83a82 kind: Event lastTimestamp: "2026-04-22T18:56:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:46Z" name: isvc-sklearn-graph-1.18a8c2afcfe8fa39 namespace: kserve-ci-e2e-test resourceVersion: "14560" uid: 5ba5404c-e0d7-440f-9c3e-8613ad21d50a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14557" uid: 9dd9cd42-ae31-4282-8980-4f8073e83a82 kind: Event lastTimestamp: "2026-04-22T18:56:46Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-04-22T18:56:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:46Z" name: isvc-sklearn-graph-1.18a8c2c5949ec7df namespace: kserve-ci-e2e-test resourceVersion: "14558" uid: 8e4610fc-8a04-4009-ad60-4703371a9981 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:55:14.473318Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13609" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:55:14Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b0123795d1 namespace: kserve-ci-e2e-test resourceVersion: "13613" uid: 4e39be1e-d507-4983-8462-108dba8640f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:14Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b01ffa7410 namespace: kserve-ci-e2e-test resourceVersion: "13625" uid: ebd4e26c-f22e-4c6d-95ac-80b1a8b72c3e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:15Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13617" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:15Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:55:15Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b0532a8ef4 namespace: kserve-ci-e2e-test resourceVersion: "13638" uid: 778883d8-ffd9-49ff-9b7e-15f020e395c7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:15Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:15Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-22T18:55:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:15Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b054c1980b namespace: kserve-ci-e2e-test resourceVersion: "13640" uid: fd77e493-4189-4504-a314-17718e4acb52 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 10.655s (10.655s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b2cfda7b7b namespace: kserve-ci-e2e-test resourceVersion: "13741" uid: 7e364f11-349a-40ae-bd9b-d9779f417465 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b2d304c482 namespace: kserve-ci-e2e-test resourceVersion: "13742" uid: e0d31335-7217-47e3-a02b-76ae09715049 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b2d3782e42 namespace: kserve-ci-e2e-test resourceVersion: "13743" uid: 9f1ad2d1-68ac-4a8c-be4f-6b4892c926fc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:30Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:30Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b3d94cf3e6 namespace: kserve-ci-e2e-test resourceVersion: "13818" uid: d2093178-7f93-4045-a82b-7ca3f4d6ef7d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1293" in 7.354s (7.354s including waiting). Image size: 1562510257 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b58fa3d766 namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: a5f6b657-bc2c-4ed2-be6a-b908c09071fc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b592d35afe namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: c6a0bc2a-2745-4b17-b113-be29eff2bc29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b5933d0856 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: fea0cdac-47aa-4609-9072-bd25fa3dbe8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b59353adfb namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 2bdb493c-45cf-4f1e-8244-c91446bfcf74 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b5965cf541 namespace: kserve-ci-e2e-test resourceVersion: "13896" uid: e4cdaee1-0b0e-4414-8b60-418c5ed8394f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:55:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:38Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b596ca9eda namespace: kserve-ci-e2e-test resourceVersion: "13897" uid: bfea6398-896f-4108-9d24-68b30b026abb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T18:55:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Readiness probe failed: dial tcp 10.132.0.18:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:55:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2b5b7df2ba8 namespace: kserve-ci-e2e-test resourceVersion: "15203" uid: a40dfbf2-d92b-44d2-9232-e5f5f1e371fb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2ce1934a50a namespace: kserve-ci-e2e-test resourceVersion: "15089" uid: 7e82d52e-791b-4cce-b3e6-b23189339a81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2ce193720a1 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: 9d5666c0-102e-43bc-9f10-bde24c609109 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs namespace: kserve-ci-e2e-test resourceVersion: "13611" uid: c86cd209-eb15-443e-8603-47d62cdea78b kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Readiness probe failed: Get "https://10.132.0.18:8643/healthz": dial tcp 10.132.0.18:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs.18a8c2ce661124ec namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: ed5dbf40-a221-44d1-a6d2-afed6c2c2cea reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-847f6cf74b namespace: kserve-ci-e2e-test resourceVersion: "13598" uid: 7680b7c1-dc8b-4218-88ff-47c3591204f9 kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-847f6cf74b-q6frs' metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:14Z" name: isvc-sklearn-graph-2-predictor-847f6cf74b.18a8c2b011d37fd4 namespace: kserve-ci-e2e-test resourceVersion: "13610" uid: 14a43b1c-a65e-4a7e-ba8f-baf50acd66a5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13596" uid: 71e0b0c6-86ff-4904-b416-15993f8c0ec6 kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-847f6cf74b from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:14Z" name: isvc-sklearn-graph-2-predictor.18a8c2affb6707f8 namespace: kserve-ci-e2e-test resourceVersion: "13599" uid: 0a88a99f-db9b-425a-8d86-8a66543bbc1e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e5ea2a7b-fc82-4eb2-94b8-7d49518ac740 kind: Event lastTimestamp: "2026-04-22T18:55:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:59Z" name: isvc-sklearn-graph-2-predictor.18a8c2b379ed466d namespace: kserve-ci-e2e-test resourceVersion: "14091" uid: 64efe1fe-0983-4109-a2e2-8e485a719af4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:29Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13597" uid: e5ea2a7b-fc82-4eb2-94b8-7d49518ac740 kind: Event lastTimestamp: "2026-04-22T18:55:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:59Z" name: isvc-sklearn-graph-2-predictor.18a8c2b37a4f6229 namespace: kserve-ci-e2e-test resourceVersion: "14092" uid: 6d17c7aa-4a57-4c8e-894c-1eef8cbf62e6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: e5ea2a7b-fc82-4eb2-94b8-7d49518ac740 kind: Event lastTimestamp: "2026-04-22T18:57:14Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:14Z" name: isvc-sklearn-graph-2-predictor.18a8c2bdf6651bda namespace: kserve-ci-e2e-test resourceVersion: "14960" uid: 9c1ef46b-f1e5-4cd6-8e88-2a286a065b48 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:56:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13800" uid: e5ea2a7b-fc82-4eb2-94b8-7d49518ac740 kind: Event lastTimestamp: "2026-04-22T18:57:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:14Z" name: isvc-sklearn-graph-2-predictor.18a8c2bdf6c5a79e namespace: kserve-ci-e2e-test resourceVersion: "14961" uid: 7b1913f0-e2c6-40e0-b5fa-d437b2a58001 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13565" uid: c572ab8c-43d9-4613-9984-cfdf3e42f5a2 kind: Event lastTimestamp: "2026-04-22T18:56:44Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:44Z" name: isvc-sklearn-graph-2.18a8c2affb3bd25f namespace: kserve-ci-e2e-test resourceVersion: "14535" uid: 85c9947f-c8b0-41cb-8392-d0e94c6faab6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:44Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14532" uid: c572ab8c-43d9-4613-9984-cfdf3e42f5a2 kind: Event lastTimestamp: "2026-04-22T18:56:44Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-04-22T18:56:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:44Z" name: isvc-sklearn-graph-2.18a8c2c51a1e188d namespace: kserve-ci-e2e-test resourceVersion: "14533" uid: a65b7b4b-5458-403a-946e-8b2df3459dd7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:55:13.874485Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13574" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-669d8d6456-msxjk to ip-10-0-134-109.ec2.internal metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:55:13Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2afee861e79 namespace: kserve-ci-e2e-test resourceVersion: "13579" uid: f17f8600-4beb-47fa-bff3-88ca6acfd4db reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13584" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:55:14Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b00812e0c4 namespace: kserve-ci-e2e-test resourceVersion: "13604" uid: a59152ec-7a25-4f7d-9e69-f7d237f3f1be reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:14Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:14Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" metadata: creationTimestamp: "2026-04-22T18:55:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:14Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b009704b21 namespace: kserve-ci-e2e-test resourceVersion: "13606" uid: dbf9b982-b86f-4a2a-acdf-bc50943ad18e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:18Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:0027065b33c40f52a3d9fc655cbe89bc4c54c77f7f755275def0561338196f29" in 4.353s (4.353s including waiting). Image size: 300951977 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b10cf382d2 namespace: kserve-ci-e2e-test resourceVersion: "13672" uid: 9746f51b-2d9b-4307-8b5c-70b754b4b467 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:18Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-04-22T18:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b111368c51 namespace: kserve-ci-e2e-test resourceVersion: "13673" uid: 3d271207-cc05-4d0e-a493-0d4d4fe717de reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:18Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:18Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-04-22T18:55:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:18Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b111b335b9 namespace: kserve-ci-e2e-test resourceVersion: "13674" uid: b3cdaf06-6e1e-49b5-ab27-6bfb5647ddde reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:23Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-04-22T18:55:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b21310bb9b namespace: kserve-ci-e2e-test resourceVersion: "13714" uid: 3cb3c998-1301-4d5b-bf96-5dd725432b4c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:43Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 20.045s (20.045s including waiting). Image size: 1306326779 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b6bde17183 namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: e95604b7-3629-42bd-a974-546cedb50f06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:43Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b6c2393b61 namespace: kserve-ci-e2e-test resourceVersion: "13943" uid: 84a44930-67a5-48bd-9aea-504bbd1b5ced reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:43Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b6c2acd3d7 namespace: kserve-ci-e2e-test resourceVersion: "13944" uid: e057ad6d-b715-4f24-982a-77af7566d4c7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:43Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-22T18:55:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b6c2b93cbc namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: fb2195d5-c8b6-4cfb-b094-fc916b40d36b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:45Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.309s (2.309s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:45Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b74c5d3950 namespace: kserve-ci-e2e-test resourceVersion: "13979" uid: a6b24259-b5f6-48b8-b8c9-82da3fbc5197 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:45Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b7500b8ec4 namespace: kserve-ci-e2e-test resourceVersion: "13980" uid: f38a739a-dcb7-4d17-a8fe-73c1b5c7b50d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:55:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:55:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:45Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b750884451 namespace: kserve-ci-e2e-test resourceVersion: "13981" uid: cac300a2-a1bf-456d-a8f0-70bed9c6fd8a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-04-22T18:55:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:56:43Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:55:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:43Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2b7aec2ebcb namespace: kserve-ci-e2e-test resourceVersion: "14498" uid: 8d1af924-5fbc-4c7d-ba14-fdc24555d6bc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2ce20ba0bb6 namespace: kserve-ci-e2e-test resourceVersion: "15153" uid: bab994c2-2d66-49af-a5ab-2e8e6e0330d8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-669d8d6456-msxjk namespace: kserve-ci-e2e-test resourceVersion: "13577" uid: 678591aa-8424-43f6-83e4-57d29c7eba30 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: isvc-xgboost-graph-predictor-669d8d6456-msxjk.18a8c2ce20bb61b1 namespace: kserve-ci-e2e-test resourceVersion: "15156" uid: 21f21b65-16a5-4273-b1d0-03a74294fa33 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-134-109.ec2.internal source: component: kubelet host: ip-10-0-134-109.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-669d8d6456 namespace: kserve-ci-e2e-test resourceVersion: "13555" uid: 2b8c6a35-cfb5-498d-ad7b-7ab314962fd1 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'Created pod: isvc-xgboost-graph-predictor-669d8d6456-msxjk' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: isvc-xgboost-graph-predictor-669d8d6456.18a8c2afee1647d6 namespace: kserve-ci-e2e-test resourceVersion: "13576" uid: 24daa562-dd37-48fc-bd39-928289d59fba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13551" uid: 8aa1e587-8c0e-4f94-a54e-572743380d63 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Scaled up replica set isvc-xgboost-graph-predictor-669d8d6456 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: isvc-xgboost-graph-predictor.18a8c2afd9997f08 namespace: kserve-ci-e2e-test resourceVersion: "13557" uid: 0bf0d31b-b424-4bf2-a1e1-024798c2a46c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13554" uid: 42e2af77-92fb-4ded-bca6-e9d600329e34 kind: Event lastTimestamp: "2026-04-22T18:56:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:28Z" name: isvc-xgboost-graph-predictor.18a8c2b3582d1088 namespace: kserve-ci-e2e-test resourceVersion: "14347" uid: 3fb2d126-29ec-4ec2-b7d7-d6ed54fa59e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13554" uid: 42e2af77-92fb-4ded-bca6-e9d600329e34 kind: Event lastTimestamp: "2026-04-22T18:56:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:28Z" name: isvc-xgboost-graph-predictor.18a8c2b3588cdb54 namespace: kserve-ci-e2e-test resourceVersion: "14348" uid: 2720c5fb-17c9-42f3-b473-c7a2d5e7f1f3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 42e2af77-92fb-4ded-bca6-e9d600329e34 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: isvc-xgboost-graph-predictor.18a8c2bdd4b476a7 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 4e6a2110-a673-4642-bf36-c91ab5d214b5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T18:56:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13785" uid: 42e2af77-92fb-4ded-bca6-e9d600329e34 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:56:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: isvc-xgboost-graph-predictor.18a8c2bdd518810a namespace: kserve-ci-e2e-test resourceVersion: "14950" uid: 7d5dabaf-9143-4531-8346-29401934b5c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13539" uid: bd082caf-95b0-4986-9318-b1007fe8f3e3 kind: Event lastTimestamp: "2026-04-22T18:56:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:53Z" name: isvc-xgboost-graph.18a8c2afd9a0f175 namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: cc07e86f-818b-42df-8a4e-da24571debf4 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14748" uid: bd082caf-95b0-4986-9318-b1007fe8f3e3 kind: Event lastTimestamp: "2026-04-22T18:56:53Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-04-22T18:56:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:53Z" name: isvc-xgboost-graph.18a8c2c7111ec24e namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: d4ede149-c0d0-40fd-8814-23c50fc821bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:57:13.246284Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-76487779f8-7cgm6 to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:57:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:57:13Z" name: model-chainer-76487779f8-7cgm6.18a8c2cbb9a32a76 namespace: kserve-ci-e2e-test resourceVersion: "14932" uid: 30e8de7c-3007-4a84-9156-6d292fe0ead1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:13Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:13Z" name: model-chainer-76487779f8-7cgm6.18a8c2cbc7531dc3 namespace: kserve-ci-e2e-test resourceVersion: "14954" uid: 9a42c7e6-38e4-481a-8b0d-c92439d16005 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:15Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14935" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:15Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:57:15Z" name: model-chainer-76487779f8-7cgm6.18a8c2cc2c3ed9eb namespace: kserve-ci-e2e-test resourceVersion: "14966" uid: b0f0715a-03ad-45b5-b8b7-c3807575b32e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:15Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:15Z" name: model-chainer-76487779f8-7cgm6.18a8c2cc2d7f7dd3 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: cca9a679-6663-40cf-a520-97ac3d2ca6ba reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:15Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-04-22T18:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:15Z" name: model-chainer-76487779f8-7cgm6.18a8c2cc308f3081 namespace: kserve-ci-e2e-test resourceVersion: "14970" uid: 6eb36630-47c6-4682-8591-cfbfa9b2a7b0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:15Z" message: Started container model-chainer metadata: creationTimestamp: "2026-04-22T18:57:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:15Z" name: model-chainer-76487779f8-7cgm6.18a8c2cc310164ef namespace: kserve-ci-e2e-test resourceVersion: "14971" uid: c4504d14-229d-40e7-9b63-3cd19bc72868 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: model-chainer-76487779f8-7cgm6.18a8c2ce136ae2f1 namespace: kserve-ci-e2e-test resourceVersion: "15062" uid: 94e066e8-b774-4ca7-992a-858455e759dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:57:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-76487779f8-7cgm6 namespace: kserve-ci-e2e-test resourceVersion: "14930" uid: d9d05e4a-c6f2-48c8-a88e-2816e26a3a9c kind: Event lastTimestamp: "2026-04-22T18:57:52Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T18:57:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:52Z" name: model-chainer-76487779f8-7cgm6.18a8c2cef4ee85e4 namespace: kserve-ci-e2e-test resourceVersion: "15502" uid: 01b3bbba-979a-4fd1-bd14-5e61eb3c29e0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-76487779f8 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: e6385990-0395-4dec-9c8e-1c8ea77e2e46 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'Created pod: model-chainer-76487779f8-7cgm6' metadata: creationTimestamp: "2026-04-22T18:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: model-chainer-76487779f8.18a8c2cbb935b446 namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: 40e0dd94-cbbb-42e0-a5b8-6f094ee62b64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: d77dcb57-4f29-4b39-8ccf-813bb5c12661 kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: Scaled up replica set model-chainer-76487779f8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:13Z" name: model-chainer.18a8c2cbb848ae57 namespace: kserve-ci-e2e-test resourceVersion: "14925" uid: d4da4bde-f67a-476e-9346-0e134241a063 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "14938" uid: 88098941-9a5b-4cfd-95e7-665f3db7780c kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-04-22T18:57:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:13Z" name: model-chainer.18a8c2cbbb8aa3a1 namespace: kserve-ci-e2e-test resourceVersion: "14941" uid: fb2681d0-03dc-41b1-922a-099b85ab2f3d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:58:23.505994Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15770" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0e291-767bbd5cdd-qwg5j to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c2dc15714ba1 namespace: kserve-ci-e2e-test resourceVersion: "15775" uid: 27fcf2ad-f0f8-40b1-bcc1-158e1526cabf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15781" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T18:58:23Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c2dc2e83c004 namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: 36e708ad-266a-4224-8b49-bdb053bb13ca reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0e291} kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T18:58:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c2dc2fcfdab7 namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: f99f0004-80ba-4036-b1d5-4c29be8bf483 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0e291} kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T18:58:24Z" message: 'Created container: sequence-graph-0e291' metadata: creationTimestamp: "2026-04-22T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:58:24Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c2dc3389d280 namespace: kserve-ci-e2e-test resourceVersion: "15799" uid: 463c03d5-0896-4ede-896f-230190e82ba0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0e291} kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T18:58:24Z" message: Started container sequence-graph-0e291 metadata: creationTimestamp: "2026-04-22T18:58:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:58:24Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c2dc33fc0482 namespace: kserve-ci-e2e-test resourceVersion: "15801" uid: 275c4bd4-beea-433d-a462-6ee2b44e7c77 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0e291} kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Stopping container sequence-graph-0e291 metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c34f491809bb namespace: kserve-ci-e2e-test resourceVersion: "19376" uid: 578608d3-2056-4e2d-9ed4-f73bc7dd22a5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0e291} kind: Pod name: sequence-graph-0e291-767bbd5cdd-qwg5j namespace: kserve-ci-e2e-test resourceVersion: "15772" uid: 2b81c6a1-d055-4e93-91a8-786d4c7267e3 kind: Event lastTimestamp: "2026-04-22T19:07:05Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:05Z" name: sequence-graph-0e291-767bbd5cdd-qwg5j.18a8c34fc5729d60 namespace: kserve-ci-e2e-test resourceVersion: "19768" uid: ec14e8fa-bc27-4eca-8aa6-53e528694cd3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0e291-767bbd5cdd namespace: kserve-ci-e2e-test resourceVersion: "15767" uid: 8c1af978-a59b-429d-b2aa-3a19ea56361b kind: Event lastTimestamp: "2026-04-22T18:58:23Z" message: 'Created pod: sequence-graph-0e291-767bbd5cdd-qwg5j' metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291-767bbd5cdd.18a8c2dc1509e8c9 namespace: kserve-ci-e2e-test resourceVersion: "15771" uid: ce439225-7e72-43d9-aeb7-439cd704d375 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15765" uid: 52fa3a30-ed90-4109-b33c-7049708c8060 kind: Event lastTimestamp: "2026-04-22T18:58:23Z" message: Scaled up replica set sequence-graph-0e291-767bbd5cdd from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291.18a8c2dc145392e8 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: dc70b8a3-33c2-4237-b01e-d7c4d65b86f5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15783" uid: 40b2ab11-e9a6-4ab9-a767-4c21bc75501b kind: Event lastTimestamp: "2026-04-22T18:58:23Z" message: InferenceGraph [sequence-graph-0e291] is Ready metadata: creationTimestamp: "2026-04-22T18:58:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:58:23Z" name: sequence-graph-0e291.18a8c2dc1768dee1 namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 317ab3dd-a697-4552-8778-5ac0c544e9b2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 5f4e4bac-019c-4986-81ce-e5621ccb480c kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: sequence-graph-0e291.18a8c2df92be8319 namespace: kserve-ci-e2e-test resourceVersion: "15904" uid: eb5e6a3a-d25b-4d11-9c10-99a540ebde8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15766" uid: 5f4e4bac-019c-4986-81ce-e5621ccb480c kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:58:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: sequence-graph-0e291.18a8c2df92fdd78f namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 69aa3ff0-887a-4940-b357-e99b56dacfb9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 5f4e4bac-019c-4986-81ce-e5621ccb480c kind: Event lastTimestamp: "2026-04-22T18:58:53Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:53Z" name: sequence-graph-0e291.18a8c2e311aa6e15 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: 5c560a21-6cb0-4233-9b57-2576e238c4d7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 5f4e4bac-019c-4986-81ce-e5621ccb480c kind: Event lastTimestamp: "2026-04-22T18:58:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:53Z" name: sequence-graph-0e291.18a8c2e31203afcc namespace: kserve-ci-e2e-test resourceVersion: "16016" uid: a3c88050-92f9-4fdd-a860-3c8854b780d9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:38.445913Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20214" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-23058-65fbffcfb8-hmthz to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:38Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35d4a707f80 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: c5bb6ab7-8dca-4cd7-b3ed-e4a684f8693f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:39Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20224" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:07:39Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:39Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35d700c0067 namespace: kserve-ci-e2e-test resourceVersion: "20237" uid: bdd9c817-3c94-48cd-bdc6-a42c054e0020 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23058} kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:07:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:39Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35d713db9d6 namespace: kserve-ci-e2e-test resourceVersion: "20239" uid: 0b165268-ab68-48cf-a441-e670a7e59a89 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23058} kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:07:39Z" message: 'Created container: sequence-graph-23058' metadata: creationTimestamp: "2026-04-22T19:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:39Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35d751ca2ad namespace: kserve-ci-e2e-test resourceVersion: "20240" uid: f80b6133-54ae-458a-9b72-944dae31e4ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23058} kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:07:39Z" message: Started container sequence-graph-23058 metadata: creationTimestamp: "2026-04-22T19:07:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:39Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35d758be8c7 namespace: kserve-ci-e2e-test resourceVersion: "20241" uid: 11df989d-da6c-40f4-8539-d2da915b1cae reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23058} kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Stopping container sequence-graph-23058 metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c35fa35ab5a8 namespace: kserve-ci-e2e-test resourceVersion: "20345" uid: 10ba1da9-a85d-47cf-99ab-442ef10606fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-23058} kind: Pod name: sequence-graph-23058-65fbffcfb8-hmthz namespace: kserve-ci-e2e-test resourceVersion: "20216" uid: 37164fb0-2462-40ed-8929-4012e1477524 kind: Event lastTimestamp: "2026-04-22T19:08:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:15Z" name: sequence-graph-23058-65fbffcfb8-hmthz.18a8c3600c0841cb namespace: kserve-ci-e2e-test resourceVersion: "20741" uid: d4a6bdb6-9fff-4a0d-b641-657fbecc31c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-23058-65fbffcfb8 namespace: kserve-ci-e2e-test resourceVersion: "20211" uid: 62c50f80-9695-4cc5-a954-f640a100e426 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: 'Created pod: sequence-graph-23058-65fbffcfb8-hmthz' metadata: creationTimestamp: "2026-04-22T19:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: sequence-graph-23058-65fbffcfb8.18a8c35d4a0fc4d2 namespace: kserve-ci-e2e-test resourceVersion: "20215" uid: 0e378b43-e9e5-43a9-afff-6d7bf200c89c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-23058 namespace: kserve-ci-e2e-test resourceVersion: "20208" uid: 67d70474-f61c-4486-a569-a9beaf46c43a kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: Scaled up replica set sequence-graph-23058-65fbffcfb8 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: sequence-graph-23058.18a8c35d4977f209 namespace: kserve-ci-e2e-test resourceVersion: "20212" uid: 88167197-8f14-48ee-acb3-86e17d7d3671 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:38Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-23058 namespace: kserve-ci-e2e-test resourceVersion: "20227" uid: 84344457-61cb-468f-af0f-0b402fc872e4 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: InferenceGraph [sequence-graph-23058] is Ready metadata: creationTimestamp: "2026-04-22T19:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:38Z" name: sequence-graph-23058.18a8c35d4ca22353 namespace: kserve-ci-e2e-test resourceVersion: "20229" uid: 30ca182e-c108-43d0-93f9-debb6ab108ec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:08:48.782651Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21038" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-ffb29-bb6b7bfb7-9tcpf to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:08:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:08:48Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36daad5e209 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 4d655b05-24b3-4f44-81c5-e7f8344c7138 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:48Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-ffb29-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:48Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36db37d2c6a namespace: kserve-ci-e2e-test resourceVersion: "21058" uid: e29f2b1b-bf37-41be-b3cd-25cb65a5d0d3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:50Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21048" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:08:50Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:08:50Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36df3e12ef0 namespace: kserve-ci-e2e-test resourceVersion: "21068" uid: a477e535-0318-4257-a738-315e31acd765 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ffb29} kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:08:50Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:50Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36df5698899 namespace: kserve-ci-e2e-test resourceVersion: "21072" uid: e6f5e7a4-78ca-499a-951e-0afb7f631baf reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ffb29} kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:08:50Z" message: 'Created container: sequence-graph-ffb29' metadata: creationTimestamp: "2026-04-22T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:50Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36df8cc42e0 namespace: kserve-ci-e2e-test resourceVersion: "21073" uid: b93829de-bce3-4dc0-b992-a9d872a8ebec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ffb29} kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:08:50Z" message: Started container sequence-graph-ffb29 metadata: creationTimestamp: "2026-04-22T19:08:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:50Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c36df943e75c namespace: kserve-ci-e2e-test resourceVersion: "21074" uid: 9dbfde15-bdd9-4cc8-a4cc-7e8394b3f517 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ffb29} kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Stopping container sequence-graph-ffb29 metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c3e0cfa46616 namespace: kserve-ci-e2e-test resourceVersion: "24652" uid: 2fc964f7-f6ec-4a46-8476-dafa5d313814 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:17:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-ffb29} kind: Pod name: sequence-graph-ffb29-bb6b7bfb7-9tcpf namespace: kserve-ci-e2e-test resourceVersion: "21040" uid: 7d4f4c46-a02f-4055-864c-69ae1b4a0f05 kind: Event lastTimestamp: "2026-04-22T19:17:31Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:31Z" name: sequence-graph-ffb29-bb6b7bfb7-9tcpf.18a8c3e18f36d594 namespace: kserve-ci-e2e-test resourceVersion: "25019" uid: 0bacd085-2446-44d2-8390-232a496fecb7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-ffb29-bb6b7bfb7 namespace: kserve-ci-e2e-test resourceVersion: "21035" uid: 6040e044-b518-4345-96fa-88b5ff7ccf51 kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: 'Created pod: sequence-graph-ffb29-bb6b7bfb7-9tcpf' metadata: creationTimestamp: "2026-04-22T19:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:48Z" name: sequence-graph-ffb29-bb6b7bfb7.18a8c36daa65ed70 namespace: kserve-ci-e2e-test resourceVersion: "21039" uid: 103d8033-9d5d-472e-ba74-cd48b724df7d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: 234b36f3-1359-4acf-b59e-dc5046b4c615 kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: Scaled up replica set sequence-graph-ffb29-bb6b7bfb7 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:48Z" name: sequence-graph-ffb29.18a8c36da8365d93 namespace: kserve-ci-e2e-test resourceVersion: "21036" uid: e6c8524e-4ebe-402f-9c6b-5563dddfa492 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "21051" uid: 11112257-2ebe-4853-9c19-fb476e6b578c kind: Event lastTimestamp: "2026-04-22T19:08:48Z" message: InferenceGraph [sequence-graph-ffb29] is Ready metadata: creationTimestamp: "2026-04-22T19:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:48Z" name: sequence-graph-ffb29.18a8c36dad082033 namespace: kserve-ci-e2e-test resourceVersion: "21054" uid: 9b305568-326b-482d-b71b-f9a663845118 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: a69e61d8-2e49-4026-ae8f-36102bbeae8a kind: Event lastTimestamp: "2026-04-22T19:09:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:09:18Z" name: sequence-graph-ffb29.18a8c37126a625ca namespace: kserve-ci-e2e-test resourceVersion: "21287" uid: 68470ad7-12ec-43b7-a394-da1eee3aef26 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "21034" uid: a69e61d8-2e49-4026-ae8f-36102bbeae8a kind: Event lastTimestamp: "2026-04-22T19:09:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:09:18Z" name: sequence-graph-ffb29.18a8c371270648e7 namespace: kserve-ci-e2e-test resourceVersion: "21288" uid: 9c3c92e0-14e4-44f9-bbcd-24be4366670f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:18:47.923396Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25896" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-352f4-56f7c7d574-mbshh to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:18:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:18:47Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c3f92a6820d3 namespace: kserve-ci-e2e-test resourceVersion: "25902" uid: 72ee003b-4a45-4ac3-9d04-ee0b53b80c3d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:48Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25905" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:18:48Z" message: Add eth0 [10.134.0.53/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:18:48Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c3f943c0c0d9 namespace: kserve-ci-e2e-test resourceVersion: "25916" uid: 202196f7-2bb9-44d3-ba82-44dcd49e4a98 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-352f4} kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:18:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:48Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c3f944ff8d58 namespace: kserve-ci-e2e-test resourceVersion: "25918" uid: 8467bde9-4d4b-4f95-bbb2-fc7a2b5e66e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-352f4} kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:18:48Z" message: 'Created container: splitter-graph-352f4' metadata: creationTimestamp: "2026-04-22T19:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:48Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c3f94847e5b2 namespace: kserve-ci-e2e-test resourceVersion: "25920" uid: c186b127-e4c7-462a-9841-c9839bb413ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-352f4} kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:18:48Z" message: Started container splitter-graph-352f4 metadata: creationTimestamp: "2026-04-22T19:18:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:48Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c3f948b9c5d0 namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: 2bbd130c-f52f-4e1a-a22e-56c40eafc936 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-352f4} kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:27:02Z" message: Stopping container splitter-graph-352f4 metadata: creationTimestamp: "2026-04-22T19:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:02Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c46c5a5cc226 namespace: kserve-ci-e2e-test resourceVersion: "29637" uid: f360343d-0a97-4e8e-85f1-436153afc723 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:27:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-352f4} kind: Pod name: splitter-graph-352f4-56f7c7d574-mbshh namespace: kserve-ci-e2e-test resourceVersion: "25899" uid: 16389b15-0804-49b8-9ef6-dc978cf25f78 kind: Event lastTimestamp: "2026-04-22T19:27:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:27:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:29Z" name: splitter-graph-352f4-56f7c7d574-mbshh.18a8c46cd0473dd5 namespace: kserve-ci-e2e-test resourceVersion: "29860" uid: 8096e03c-1eb6-419c-89f4-ec3dd4c5e2ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-352f4-56f7c7d574 namespace: kserve-ci-e2e-test resourceVersion: "25893" uid: e5b819a7-acdb-47f8-9901-81ac9f18e82f kind: Event lastTimestamp: "2026-04-22T19:18:47Z" message: 'Created pod: splitter-graph-352f4-56f7c7d574-mbshh' metadata: creationTimestamp: "2026-04-22T19:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:47Z" name: splitter-graph-352f4-56f7c7d574.18a8c3f929f9bb06 namespace: kserve-ci-e2e-test resourceVersion: "25898" uid: 35360311-c46f-4c9e-8152-e00beabee17a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25891" uid: e9403e98-3418-4c19-ae5d-e22abfa1688a kind: Event lastTimestamp: "2026-04-22T19:18:47Z" message: Scaled up replica set splitter-graph-352f4-56f7c7d574 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:47Z" name: splitter-graph-352f4.18a8c3f9290f0c47 namespace: kserve-ci-e2e-test resourceVersion: "25894" uid: f64aeb7e-05ea-4ff0-af05-e2f99043ae07 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25908" uid: 602a9513-5992-4ecb-b9d3-35b5c99dd59e kind: Event lastTimestamp: "2026-04-22T19:18:47Z" message: InferenceGraph [splitter-graph-352f4] is Ready metadata: creationTimestamp: "2026-04-22T19:18:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:47Z" name: splitter-graph-352f4.18a8c3f92dbd43de namespace: kserve-ci-e2e-test resourceVersion: "25911" uid: a37450b1-d7f2-428a-b581-e4f42c9991f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 993d9249-2050-43c8-9fbf-ecf649e450b7 kind: Event lastTimestamp: "2026-04-22T19:19:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:02Z" name: splitter-graph-352f4.18a8c3fca7897f90 namespace: kserve-ci-e2e-test resourceVersion: "26034" uid: 78a0f4a4-b8f5-4a8f-b962-3a8149ef23f1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:19:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25892" uid: 993d9249-2050-43c8-9fbf-ecf649e450b7 kind: Event lastTimestamp: "2026-04-22T19:19:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:19:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:02Z" name: splitter-graph-352f4.18a8c3fca7ee9b0c namespace: kserve-ci-e2e-test resourceVersion: "26036" uid: 7be59c2a-2f19-45bb-9a82-a1c94c119e02 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:19:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 993d9249-2050-43c8-9fbf-ecf649e450b7 kind: Event lastTimestamp: "2026-04-22T19:19:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:17Z" name: splitter-graph-352f4.18a8c4002698cba9 namespace: kserve-ci-e2e-test resourceVersion: "26429" uid: ac8dc892-443b-45dd-90ff-d0a2c220e5f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:19:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-352f4 namespace: kserve-ci-e2e-test resourceVersion: "26035" uid: 993d9249-2050-43c8-9fbf-ecf649e450b7 kind: Event lastTimestamp: "2026-04-22T19:19:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:19:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:17Z" name: splitter-graph-352f4.18a8c40026f63fb2 namespace: kserve-ci-e2e-test resourceVersion: "26431" uid: 825cf631-ffb4-4c69-8ccc-fd8b7d2f72e5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:17:37.653534Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25082" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-c8908-6bc4bd4ff9-82b7p to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:17:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:17:37Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e8cdff6022 namespace: kserve-ci-e2e-test resourceVersion: "25087" uid: 761d85f5-2c1f-44b7-8536-7ccd5d886049 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:37Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-c8908-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:37Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e8d93e4652 namespace: kserve-ci-e2e-test resourceVersion: "25097" uid: 91da2d48-e7fe-4f9a-81d3-67a68bb07eda reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:38Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25090" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:38Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:17:38Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e90b48e4d0 namespace: kserve-ci-e2e-test resourceVersion: "25105" uid: 8fdfd689-ff03-45e6-bdd9-f4a68770ee9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c8908} kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:38Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e90c915eb1 namespace: kserve-ci-e2e-test resourceVersion: "25107" uid: 13dc333b-db4e-40b1-b29e-f5355a15de45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c8908} kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:38Z" message: 'Created container: splitter-graph-c8908' metadata: creationTimestamp: "2026-04-22T19:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:38Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e90fad675c namespace: kserve-ci-e2e-test resourceVersion: "25108" uid: 52043bd3-2220-4983-b8d8-156ed768900c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c8908} kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:38Z" message: Started container splitter-graph-c8908 metadata: creationTimestamp: "2026-04-22T19:17:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:38Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3e9101e6d0d namespace: kserve-ci-e2e-test resourceVersion: "25109" uid: a246bd69-d00f-4fee-a842-8427484213e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c8908} kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Stopping container splitter-graph-c8908 metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:47Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3eb25bd2613 namespace: kserve-ci-e2e-test resourceVersion: "25204" uid: 89603174-d7e8-4dbc-93be-9eb3606a07bf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-c8908} kind: Pod name: splitter-graph-c8908-6bc4bd4ff9-82b7p namespace: kserve-ci-e2e-test resourceVersion: "25085" uid: b16177f8-1c92-4b97-8749-13a66888dea4 kind: Event lastTimestamp: "2026-04-22T19:18:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:15Z" name: splitter-graph-c8908-6bc4bd4ff9-82b7p.18a8c3ebc3adf913 namespace: kserve-ci-e2e-test resourceVersion: "25597" uid: 26f82b3c-c69e-4976-9a56-f4709cc21e01 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-c8908-6bc4bd4ff9 namespace: kserve-ci-e2e-test resourceVersion: "25079" uid: 94bff0a9-59f8-4585-ba16-eea53e3f3426 kind: Event lastTimestamp: "2026-04-22T19:17:37Z" message: 'Created pod: splitter-graph-c8908-6bc4bd4ff9-82b7p' metadata: creationTimestamp: "2026-04-22T19:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:37Z" name: splitter-graph-c8908-6bc4bd4ff9.18a8c3e8cd954497 namespace: kserve-ci-e2e-test resourceVersion: "25084" uid: bb354604-4e42-40bc-a15d-80bbfc441197 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-c8908 namespace: kserve-ci-e2e-test resourceVersion: "25076" uid: abcf25a2-a80e-4e89-9e9a-be0e40b162e2 kind: Event lastTimestamp: "2026-04-22T19:17:37Z" message: Scaled up replica set splitter-graph-c8908-6bc4bd4ff9 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:37Z" name: splitter-graph-c8908.18a8c3e8ccb85aca namespace: kserve-ci-e2e-test resourceVersion: "25080" uid: 6807640c-3f51-4a36-ae78-af5745f89912 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-c8908 namespace: kserve-ci-e2e-test resourceVersion: "25093" uid: d16934d8-d319-47ed-ae76-50f8b75050ab kind: Event lastTimestamp: "2026-04-22T19:17:37Z" message: InferenceGraph [splitter-graph-c8908] is Ready metadata: creationTimestamp: "2026-04-22T19:17:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:37Z" name: splitter-graph-c8908.18a8c3e8d11d3768 namespace: kserve-ci-e2e-test resourceVersion: "25095" uid: dce407e1-1e01-4c42-ac0d-bf8d3d2bc605 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:57:23.487589Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:57:23Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce1c111f22 namespace: kserve-ci-e2e-test resourceVersion: "15117" uid: ecb7d47d-55c4-4737-b19c-ef88dce04d0d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-0e291-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:23Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce272a67a9 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 3ce20523-b21d-4774-911a-de1a69aba10b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15124" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce58a2d7b9 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: f8b5caec-2f7d-49ac-8e17-f36fc6042aad reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce59e700be namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 010e7ce4-7d5d-4c87-9fd9-762b75952a4d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce5ca87d88 namespace: kserve-ci-e2e-test resourceVersion: "15190" uid: bcd78dd5-4210-4e95-a8fb-7f0cc3f684ec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce5d177a64 namespace: kserve-ci-e2e-test resourceVersion: "15191" uid: 5bcce95b-667f-483c-8093-d5fd4a67f68c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce5d22aa3d namespace: kserve-ci-e2e-test resourceVersion: "15192" uid: 18d5fea5-b772-4929-ac2a-ac70960d286a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce60378bcf namespace: kserve-ci-e2e-test resourceVersion: "15195" uid: 4bad53e1-5d5f-4f3e-9de6-d61b47a953ea reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:57:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:57:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:24Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ce60f259f1 namespace: kserve-ci-e2e-test resourceVersion: "15197" uid: b64be863-7add-43d8-b1be-9ee85a35eaeb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:57:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T18:58:02Z" message: 'Readiness probe failed: dial tcp 10.132.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:57:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:58:02Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c2ceb347674b namespace: kserve-ci-e2e-test resourceVersion: "15583" uid: c806b1e8-4269-41fd-a603-d1bcb7093dc9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c34f4ee1d6db namespace: kserve-ci-e2e-test resourceVersion: "19402" uid: 079ac660-c93c-4d8c-827d-5d5e8360af7d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: 0a7e6e64-c5d9-42b0-9de6-4880b17a1529 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt.18a8c34f4ee26eb6 namespace: kserve-ci-e2e-test resourceVersion: "19406" uid: e0d010ae-e1e2-4a7b-984a-0075de956249 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0e291-predictor-5b45dc76dd namespace: kserve-ci-e2e-test resourceVersion: "15092" uid: d58d30c0-3380-436e-b075-625c8514ca97 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: 'Created pod: success-200-isvc-0e291-predictor-5b45dc76dd-4q8jt' metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:23Z" name: success-200-isvc-0e291-predictor-5b45dc76dd.18a8c2ce1ba47afc namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 0c1e8a14-d2ea-4e6d-a974-13139d0f8596 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15088" uid: c57f4b35-2ae9-4cac-82c7-81508ad4d485 kind: Event lastTimestamp: "2026-04-22T18:57:23Z" message: Scaled up replica set success-200-isvc-0e291-predictor-5b45dc76dd from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:23Z" name: success-200-isvc-0e291-predictor.18a8c2ce19eee68c namespace: kserve-ci-e2e-test resourceVersion: "15095" uid: 240816e1-099b-4026-b005-fcf5fa15265b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: a5414b1f-bfdf-4d1a-b219-23d82cf0aadf kind: Event lastTimestamp: "2026-04-22T18:57:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:53Z" name: success-200-isvc-0e291-predictor.18a8c2d1986c6b44 namespace: kserve-ci-e2e-test resourceVersion: "15510" uid: 5e381490-c527-4bdf-90e6-ad1b5e36b1e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: a5414b1f-bfdf-4d1a-b219-23d82cf0aadf kind: Event lastTimestamp: "2026-04-22T18:57:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:53Z" name: success-200-isvc-0e291-predictor.18a8c2d198cac1e8 namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: 605e1fd2-3928-4765-b0fd-88b87428dfa7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: a5414b1f-bfdf-4d1a-b219-23d82cf0aadf kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: success-200-isvc-0e291-predictor.18a8c2d896793ef1 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 61c5fa00-7be2-4604-8790-eb87d7a1f738 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:58:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0e291-predictor namespace: kserve-ci-e2e-test resourceVersion: "15356" uid: a5414b1f-bfdf-4d1a-b219-23d82cf0aadf kind: Event lastTimestamp: "2026-04-22T18:58:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:38Z" name: success-200-isvc-0e291-predictor.18a8c2d896da32cc namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: b8e825a5-31ae-42ee-a526-6bd469e640a7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:57:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15067" uid: 07c78411-df6c-43a3-9524-3c35e79128a6 kind: Event lastTimestamp: "2026-04-22T18:58:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:57:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:58:12Z" name: success-200-isvc-0e291.18a8c2ce19c3594d namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: dd834952-4e95-4628-8944-97a1b347d66e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0e291 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 07c78411-df6c-43a3-9524-3c35e79128a6 kind: Event lastTimestamp: "2026-04-22T18:58:12Z" message: InferenceService [success-200-isvc-0e291] is Ready metadata: creationTimestamp: "2026-04-22T18:58:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:58:12Z" name: success-200-isvc-0e291.18a8c2d96d8661d3 namespace: kserve-ci-e2e-test resourceVersion: "15667" uid: f2f0da79-4c53-4b15-9523-919934cacd0c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:06:38.431883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f51531914 namespace: kserve-ci-e2e-test resourceVersion: "19417" uid: c374ae47-70a4-4262-8507-0e490736628a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19432" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f6abe4f04 namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 508d8726-3421-4687-8133-4677997c07a7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f6bff1e4c namespace: kserve-ci-e2e-test resourceVersion: "19473" uid: 72d41ff5-db64-4c03-8571-3f2ea8ef5083 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f6edffcb1 namespace: kserve-ci-e2e-test resourceVersion: "19476" uid: 16fa0064-401f-46ec-bf34-902689ffc19f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f6f57f825 namespace: kserve-ci-e2e-test resourceVersion: "19477" uid: e9a70f3f-53d4-45e8-a27c-7825728df970 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f6f651fca namespace: kserve-ci-e2e-test resourceVersion: "19478" uid: 17289eb3-bef9-4ff7-bbfd-6f4bd50ecbf7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f723bb82d namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: a301c20b-95e0-4609-af2b-5a9aced503f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34f72c8c02d namespace: kserve-ci-e2e-test resourceVersion: "19480" uid: dfbd3d5e-0da1-4b67-b997-21dbebddfe74 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:07:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:06:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:16Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c34fcc15ed45 namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: 2e15c95b-1667-4cc9-9f60-bd7f1fbd60c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c35fa9e5d257 namespace: kserve-ci-e2e-test resourceVersion: "20378" uid: 93552d82-8557-4990-a0f2-bebba511fbb4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c35fa9e690c5 namespace: kserve-ci-e2e-test resourceVersion: "20380" uid: 4cfcd7c5-6782-47f9-9203-a55ac04e3287 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv namespace: kserve-ci-e2e-test resourceVersion: "19414" uid: 776e5fb7-2c36-4dc6-8640-05bc68ff731a kind: Event lastTimestamp: "2026-04-22T19:07:51Z" message: 'Readiness probe failed: Get "https://10.132.0.22:8643/healthz": dial tcp 10.132.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:51Z" name: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv.18a8c360542692f1 namespace: kserve-ci-e2e-test resourceVersion: "20483" uid: c1b991c5-67ea-4ba9-b787-41daa24fdbd8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-23058-predictor-68c8c4c7bb namespace: kserve-ci-e2e-test resourceVersion: "19401" uid: 18ba7dc6-bcb4-4229-9129-a3d5498eef9f kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Created pod: success-200-isvc-23058-predictor-68c8c4c7bb-7j2rv' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor-68c8c4c7bb.18a8c34f50ec4709 namespace: kserve-ci-e2e-test resourceVersion: "19415" uid: 981d6e54-e4f6-4443-b454-763c7b5debe3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19398" uid: a6bc663a-ac04-43fd-afb1-af43032136fd kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: Scaled up replica set success-200-isvc-23058-predictor-68c8c4c7bb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058-predictor.18a8c34f4f095221 namespace: kserve-ci-e2e-test resourceVersion: "19404" uid: 6b289afa-bf8e-4f63-a96b-40b76c4e2c46 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: 05f46508-bd5c-4c44-a78a-507e6667c8a9 kind: Event lastTimestamp: "2026-04-22T19:07:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:08Z" name: success-200-isvc-23058-predictor.18a8c352cd76837e namespace: kserve-ci-e2e-test resourceVersion: "19792" uid: df6262c4-3394-46cd-8dc5-aabfbbc454dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:06:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19400" uid: 05f46508-bd5c-4c44-a78a-507e6667c8a9 kind: Event lastTimestamp: "2026-04-22T19:07:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:08Z" name: success-200-isvc-23058-predictor.18a8c352cdd20ab4 namespace: kserve-ci-e2e-test resourceVersion: "19794" uid: 6380a606-4ae8-453e-b1b0-c6dc982a2375 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 05f46508-bd5c-4c44-a78a-507e6667c8a9 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: success-200-isvc-23058-predictor.18a8c359cb797839 namespace: kserve-ci-e2e-test resourceVersion: "20222" uid: dba323de-395d-46ff-bef7-d4e87b48c582 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-23058-predictor namespace: kserve-ci-e2e-test resourceVersion: "19632" uid: 05f46508-bd5c-4c44-a78a-507e6667c8a9 kind: Event lastTimestamp: "2026-04-22T19:07:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:38Z" name: success-200-isvc-23058-predictor.18a8c359cbdbcbe5 namespace: kserve-ci-e2e-test resourceVersion: "20225" uid: a50f4538-4dc4-4795-ab8f-f2c1bf24258f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: 03a784ce-371e-488f-b3e5-930af7a3f215 kind: Event lastTimestamp: "2026-04-22T19:07:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:26Z" name: success-200-isvc-23058.18a8c34f4ee186f3 namespace: kserve-ci-e2e-test resourceVersion: "20103" uid: 8bb43087-5224-49f6-84dc-a780c82b2ab8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: 03a784ce-371e-488f-b3e5-930af7a3f215 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'Failed to update status for InferenceService "success-200-isvc-23058": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-23058": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058.18a8c34f4f4baac2 namespace: kserve-ci-e2e-test resourceVersion: "19407" uid: 91244e69-b4d7-440e-b8ae-6c33a943be96 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "19378" uid: 03a784ce-371e-488f-b3e5-930af7a3f215 kind: Event lastTimestamp: "2026-04-22T19:06:38Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-23058": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:38Z" name: success-200-isvc-23058.18a8c34f4f4c0ad2 namespace: kserve-ci-e2e-test resourceVersion: "19408" uid: 2b2d886c-dba7-4974-acd9-81ff27b244a5 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-23058 namespace: kserve-ci-e2e-test resourceVersion: "20101" uid: 03a784ce-371e-488f-b3e5-930af7a3f215 kind: Event lastTimestamp: "2026-04-22T19:07:26Z" message: InferenceService [success-200-isvc-23058] is Ready metadata: creationTimestamp: "2026-04-22T19:07:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:26Z" name: success-200-isvc-23058.18a8c35a8610e6fd namespace: kserve-ci-e2e-test resourceVersion: "20102" uid: 4d50bdb4-1451-4a24-9bf7-bc86d5bbc72b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:17:47.848830Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25239" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb2daf164d namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: c9ac4a7f-6834-46a5-ba20-a1f7f15ff2ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-352f4-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb372c9268 namespace: kserve-ci-e2e-test resourceVersion: "25291" uid: 2ef22bd3-dbc7-4ba2-aba1-c5dc4baf01c5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25249" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb6aabbb28 namespace: kserve-ci-e2e-test resourceVersion: "25318" uid: 17a80706-f27f-41e1-8b0a-79af92894d50 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb6bdc3a06 namespace: kserve-ci-e2e-test resourceVersion: "25320" uid: 888382c4-8908-432b-b8b3-a40f0108f04f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb6ed9ac3d namespace: kserve-ci-e2e-test resourceVersion: "25321" uid: 75fbfd5b-0ee3-4523-8a76-fd72eb465600 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb6f54cf6f namespace: kserve-ci-e2e-test resourceVersion: "25322" uid: 4964b858-8e50-40fb-be6f-65615d5886a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb6f6164ce namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: 06557c1d-4b95-44f9-889a-7053650ad211 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:48Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb722db408 namespace: kserve-ci-e2e-test resourceVersion: "25324" uid: 884418c8-3669-4595-96de-91066d874ac9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:17:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:49Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3eb72a43578 namespace: kserve-ci-e2e-test resourceVersion: "25325" uid: 0f6d78e0-54ea-4c55-831f-4f08b75cdf15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:17:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:18:26Z" message: 'Readiness probe failed: dial tcp 10.132.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:17:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:26Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c3ebc283dbf5 namespace: kserve-ci-e2e-test resourceVersion: "25696" uid: 7d6504d6-c2b7-4b00-b2ee-247f7600daa2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:27:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:02Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c46c5f641654 namespace: kserve-ci-e2e-test resourceVersion: "29651" uid: 7199fc52-1bb0-465d-a652-0088155980a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:27:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: 815909cd-48dc-47d0-80b7-aa2812993f0a kind: Event lastTimestamp: "2026-04-22T19:27:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:27:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:27:02Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx.18a8c46c5f649c80 namespace: kserve-ci-e2e-test resourceVersion: "29652" uid: 2dfa4b20-7db2-40ca-a27c-272d4657c3a4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-352f4-predictor-5f6bdb8c66 namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: 2615e0b1-1462-4463-bb4f-e000d02d30e4 kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'Created pod: success-200-isvc-352f4-predictor-5f6bdb8c66-j6kcx' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-352f4-predictor-5f6bdb8c66.18a8c3eb2d2bbef6 namespace: kserve-ci-e2e-test resourceVersion: "25241" uid: 139b6acf-bc71-4a7f-a563-4211fc1edaac reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 96a24c6e-389e-4811-b962-7b47ba8db90a kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Scaled up replica set success-200-isvc-352f4-predictor-5f6bdb8c66 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-352f4-predictor.18a8c3eb2b7752fb namespace: kserve-ci-e2e-test resourceVersion: "25231" uid: d18df822-927f-40ea-b12c-7d8d7701e94c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: b86e5d27-56f9-42a9-b743-45b22df446e0 kind: Event lastTimestamp: "2026-04-22T19:18:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:17Z" name: success-200-isvc-352f4-predictor.18a8c3eeaa50ab37 namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: 2a4c781b-b8c1-4826-aae8-2f3296afa79b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:18:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: b86e5d27-56f9-42a9-b743-45b22df446e0 kind: Event lastTimestamp: "2026-04-22T19:18:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:17Z" name: success-200-isvc-352f4-predictor.18a8c3eeaab32741 namespace: kserve-ci-e2e-test resourceVersion: "25626" uid: 81b3ba5a-704e-4e0b-bfe8-5b1fda962431 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: b86e5d27-56f9-42a9-b743-45b22df446e0 kind: Event lastTimestamp: "2026-04-22T19:19:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:02Z" name: success-200-isvc-352f4-predictor.18a8c3f5a8847f82 namespace: kserve-ci-e2e-test resourceVersion: "26032" uid: c6066aa3-4c9a-492b-9417-adedc2ecba8b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:18:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-352f4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25466" uid: b86e5d27-56f9-42a9-b743-45b22df446e0 kind: Event lastTimestamp: "2026-04-22T19:19:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:18:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:19:02Z" name: success-200-isvc-352f4-predictor.18a8c3f5a8e860d7 namespace: kserve-ci-e2e-test resourceVersion: "26033" uid: ee02753d-63ee-4c96-a4dc-07d187a5e538 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: ad8d9bac-19c0-4ae7-aeda-c9e8bf3ffd3e kind: Event lastTimestamp: "2026-04-22T19:18:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:36Z" name: success-200-isvc-352f4.18a8c3eb2b5005ec namespace: kserve-ci-e2e-test resourceVersion: "25792" uid: 5f270a2a-980c-43a0-b8c3-7d8a64c157a0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: ad8d9bac-19c0-4ae7-aeda-c9e8bf3ffd3e kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-352f4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-352f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-352f4.18a8c3eb2b9dfd3c namespace: kserve-ci-e2e-test resourceVersion: "25233" uid: 0d5c12de-9107-44b5-a540-de98156b0bf9 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25208" uid: ad8d9bac-19c0-4ae7-aeda-c9e8bf3ffd3e kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-352f4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-352f4.18a8c3eb2b9e56df namespace: kserve-ci-e2e-test resourceVersion: "25235" uid: cae68b2b-5ee1-40da-86c1-8e1fe5b101ac reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-352f4 namespace: kserve-ci-e2e-test resourceVersion: "25783" uid: ad8d9bac-19c0-4ae7-aeda-c9e8bf3ffd3e kind: Event lastTimestamp: "2026-04-22T19:18:36Z" message: InferenceService [success-200-isvc-352f4] is Ready metadata: creationTimestamp: "2026-04-22T19:18:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:36Z" name: success-200-isvc-352f4.18a8c3f67ce8b41a namespace: kserve-ci-e2e-test resourceVersion: "25785" uid: 6cbc3b4a-73d2-40dd-9391-32d927943391 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:06:02.521991Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18995" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-50a5e-predictor-6b875d9454-7mxtj to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c346f4eda5a7 namespace: kserve-ci-e2e-test resourceVersion: "19008" uid: 1e226ebc-fc1d-4462-b1d1-3c66b330abd0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "19019" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3470dddd8b6 namespace: kserve-ci-e2e-test resourceVersion: "19051" uid: 9c978082-63fa-4020-8cab-587a4411fe0e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3470f1ab467 namespace: kserve-ci-e2e-test resourceVersion: "19054" uid: b7c6fc23-1c46-49b4-8e4e-f7b56bd89e7d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c34712a4529c namespace: kserve-ci-e2e-test resourceVersion: "19055" uid: 80bd5f07-9fce-4a51-ab09-42d4900c878a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3471313ec52 namespace: kserve-ci-e2e-test resourceVersion: "19056" uid: 6319b208-25db-4983-8d54-11154079e61c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c34713213a21 namespace: kserve-ci-e2e-test resourceVersion: "19057" uid: 137ee747-e002-4fcf-908c-e64e52a847f5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c34717143b5c namespace: kserve-ci-e2e-test resourceVersion: "19060" uid: 3610af2d-1012-4be4-9fa6-99788113a945 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:03Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c34717897c40 namespace: kserve-ci-e2e-test resourceVersion: "19061" uid: 4dcf1733-6058-4d5c-b432-10f51c6df3e8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:06:40Z" message: 'Readiness probe failed: dial tcp 10.132.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:40Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3476438b257 namespace: kserve-ci-e2e-test resourceVersion: "19506" uid: 86271af9-dff8-46c1-be2c-57c8d5786c4c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3574defda92 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: ff9aff59-31b2-417d-9926-65e324acfcdc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c3574df0564f namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: af492987-1eeb-4701-8c71-387a276a350e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj namespace: kserve-ci-e2e-test resourceVersion: "18998" uid: 7f80fa95-499a-400e-bb2b-8590ce422211 kind: Event lastTimestamp: "2026-04-22T19:07:15Z" message: 'Readiness probe failed: Get "https://10.132.0.21:8643/healthz": dial tcp 10.132.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:15Z" name: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj.18a8c357ec664f61 namespace: kserve-ci-e2e-test resourceVersion: "19985" uid: 13c76703-b323-4065-99f3-acd0f02650d0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-50a5e-predictor-6b875d9454 namespace: kserve-ci-e2e-test resourceVersion: "18980" uid: a9e8d6c0-fc39-4b78-8a90-b036be45a374 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'Created pod: success-200-isvc-50a5e-predictor-6b875d9454-7mxtj' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e-predictor-6b875d9454.18a8c346f481d603 namespace: kserve-ci-e2e-test resourceVersion: "18999" uid: a0e4ff10-0111-4f89-ad4e-06901da291de reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18972" uid: e8aad3b9-d3ae-47e6-9953-34cc691c7058 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Scaled up replica set success-200-isvc-50a5e-predictor-6b875d9454 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e-predictor.18a8c346f14b4620 namespace: kserve-ci-e2e-test resourceVersion: "18983" uid: d9c6dd9c-ed73-449d-9f1d-cf3a8ba3aa62 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 528f0c30-3ec9-4823-b51e-95adb99fb777 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: success-200-isvc-50a5e-predictor.18a8c34a6fea45ca namespace: kserve-ci-e2e-test resourceVersion: "19715" uid: 0258535a-63af-4946-8778-760ed28a07e1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:06:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "18979" uid: 528f0c30-3ec9-4823-b51e-95adb99fb777 kind: Event lastTimestamp: "2026-04-22T19:07:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:06:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:02Z" name: success-200-isvc-50a5e-predictor.18a8c34a7030b7c3 namespace: kserve-ci-e2e-test resourceVersion: "19718" uid: ca98c943-b3d9-4c97-8c5c-a1987ec505b5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: 528f0c30-3ec9-4823-b51e-95adb99fb777 kind: Event lastTimestamp: "2026-04-22T19:06:32Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:32Z" name: success-200-isvc-50a5e-predictor.18a8c34def28d7fa namespace: kserve-ci-e2e-test resourceVersion: "19308" uid: ce24e74a-beb4-4a85-8cdb-5979ab94f140 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-50a5e-predictor namespace: kserve-ci-e2e-test resourceVersion: "19194" uid: 528f0c30-3ec9-4823-b51e-95adb99fb777 kind: Event lastTimestamp: "2026-04-22T19:06:32Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:06:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:06:32Z" name: success-200-isvc-50a5e-predictor.18a8c34def881070 namespace: kserve-ci-e2e-test resourceVersion: "19310" uid: 459df73e-1d0f-4fe3-8fc4-63636e31a9d6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18958" uid: 8b1e11f9-a2b9-4060-8aca-3c1c6de2decc kind: Event lastTimestamp: "2026-04-22T19:06:50Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:50Z" name: success-200-isvc-50a5e.18a8c346f14b7fd1 namespace: kserve-ci-e2e-test resourceVersion: "19603" uid: 781f9e95-720c-4ea6-af56-ee4e4435c921 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18958" uid: 8b1e11f9-a2b9-4060-8aca-3c1c6de2decc kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-50a5e": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-50a5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e.18a8c346f1dd589e namespace: kserve-ci-e2e-test resourceVersion: "18987" uid: cebef6c0-8ad3-45e4-a6e0-466fde44561d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "18958" uid: 8b1e11f9-a2b9-4060-8aca-3c1c6de2decc kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-50a5e": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-50a5e.18a8c346f1dda6f8 namespace: kserve-ci-e2e-test resourceVersion: "18989" uid: 0d6d7052-6968-4284-b196-6d15d612615e reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:50Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-50a5e namespace: kserve-ci-e2e-test resourceVersion: "19601" uid: 8b1e11f9-a2b9-4060-8aca-3c1c6de2decc kind: Event lastTimestamp: "2026-04-22T19:06:50Z" message: InferenceService [success-200-isvc-50a5e] is Ready metadata: creationTimestamp: "2026-04-22T19:06:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:06:50Z" name: success-200-isvc-50a5e.18a8c3521e1f8ae0 namespace: kserve-ci-e2e-test resourceVersion: "19602" uid: e6b6b274-0a06-413e-a8c3-efd27f8ea994 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:56:47.605005Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14625" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-70f3d-predictor-f4754d89d-rtwtg to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c5c14c5dce namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 970c579f-a69f-4ef9-b476-a0934ee0e73f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-70f3d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c5cd53d352 namespace: kserve-ci-e2e-test resourceVersion: "14670" uid: 08c541c5-679f-4890-82a9-08125f3043e9 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14644" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c609eaca6f namespace: kserve-ci-e2e-test resourceVersion: "14690" uid: 22a6ab69-b7ac-4c07-85f8-8b518271fb38 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c60b258732 namespace: kserve-ci-e2e-test resourceVersion: "14692" uid: 370bfe99-ec0f-4cbd-b163-66e45ffbd69c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c60e081d22 namespace: kserve-ci-e2e-test resourceVersion: "14694" uid: 06872ff8-1ab5-45bf-8d0d-65cb0e83db3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c60e781645 namespace: kserve-ci-e2e-test resourceVersion: "14695" uid: 572d8791-2a37-4084-9300-921053613259 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c60e8448da namespace: kserve-ci-e2e-test resourceVersion: "14696" uid: dd63418c-43ca-41bc-8288-754cf7d3fb68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c61237b0c1 namespace: kserve-ci-e2e-test resourceVersion: "14698" uid: c54b5d51-d898-4a56-9e7f-483120652123 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:56:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:48Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c612a548b9 namespace: kserve-ci-e2e-test resourceVersion: "14699" uid: 07c86cbd-eedc-4a15-88cf-c71976afb228 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:56:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T18:57:25Z" message: 'Readiness probe failed: dial tcp 10.132.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:56:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:25Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c2c64b801be2 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 36b87dc0-c446-47a0-bdae-6d1211c49334 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c346f242c31a namespace: kserve-ci-e2e-test resourceVersion: "18991" uid: c7065732-f092-4571-979a-e0f10666e2ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: ea3777a4-31a2-460c-b453-d828f465a603 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg.18a8c346f2437e01 namespace: kserve-ci-e2e-test resourceVersion: "18992" uid: bbc70147-7ac5-4eef-8c29-bb38a0c86409 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-70f3d-predictor-f4754d89d namespace: kserve-ci-e2e-test resourceVersion: "14607" uid: a10ca692-708e-4551-9334-acd72580f414 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'Created pod: success-200-isvc-70f3d-predictor-f4754d89d-rtwtg' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d-predictor-f4754d89d.18a8c2c5c0e96676 namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: f306470c-b55d-4add-a3d8-99820e46be2e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14604" uid: 3ddfceb7-4f98-41dd-8408-d713960a6e42 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Scaled up replica set success-200-isvc-70f3d-predictor-f4754d89d from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d-predictor.18a8c2c5bdde66f6 namespace: kserve-ci-e2e-test resourceVersion: "14609" uid: ba98338b-df5a-492b-a5c3-8625e5d2d148 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: eeb6f9a2-5cc8-4d81-a4d9-615e9e6b3a4e kind: Event lastTimestamp: "2026-04-22T18:57:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:17Z" name: success-200-isvc-70f3d-predictor.18a8c2c93cfeda99 namespace: kserve-ci-e2e-test resourceVersion: "14992" uid: cc7357f3-0182-4cec-81cc-29ad74b92d9a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14606" uid: eeb6f9a2-5cc8-4d81-a4d9-615e9e6b3a4e kind: Event lastTimestamp: "2026-04-22T18:57:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:57:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:17Z" name: success-200-isvc-70f3d-predictor.18a8c2c93d36394a namespace: kserve-ci-e2e-test resourceVersion: "14993" uid: f42b470c-73f3-4150-b4d2-23955cdef944 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: eeb6f9a2-5cc8-4d81-a4d9-615e9e6b3a4e kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: success-200-isvc-70f3d-predictor.18a8c2d03ae77c82 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 48cbef12-e671-470b-9981-e697582a37b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T18:57:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-70f3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14836" uid: eeb6f9a2-5cc8-4d81-a4d9-615e9e6b3a4e kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:57:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: success-200-isvc-70f3d-predictor.18a8c2d03b49e4f6 namespace: kserve-ci-e2e-test resourceVersion: "15437" uid: f2721f6e-7bb7-41d2-8704-26dec0cef18a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 8a4b8cbd-dbaa-4ad3-b9e9-edad3f3f3d62 kind: Event lastTimestamp: "2026-04-22T18:57:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:36Z" name: success-200-isvc-70f3d.18a8c2c5bdded07c namespace: kserve-ci-e2e-test resourceVersion: "15328" uid: e3e9bd18-2f72-45a4-a820-857ef20985a8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 8a4b8cbd-dbaa-4ad3-b9e9-edad3f3f3d62 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-70f3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-70f3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d.18a8c2c5be69ff3d namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: 55792265-ee03-443e-bf68-6ef5938eea13 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "14582" uid: 8a4b8cbd-dbaa-4ad3-b9e9-edad3f3f3d62 kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-70f3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-70f3d.18a8c2c5be6a4d33 namespace: kserve-ci-e2e-test resourceVersion: "14614" uid: bde03404-2a79-42ea-a13f-5d5a6f0c71d4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15324" uid: 8a4b8cbd-dbaa-4ad3-b9e9-edad3f3f3d62 kind: Event lastTimestamp: "2026-04-22T18:57:35Z" message: InferenceService [success-200-isvc-70f3d] is Ready metadata: creationTimestamp: "2026-04-22T18:57:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:35Z" name: success-200-isvc-70f3d.18a8c2d105b24582 namespace: kserve-ci-e2e-test resourceVersion: "15325" uid: 52c1152b-0914-4df6-a79d-77be31905a01 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:55:13.210533Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13476" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-9844b-predictor-758ffd9676-tf4ql to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2afc6f365b0 namespace: kserve-ci-e2e-test resourceVersion: "13482" uid: f97a1bd4-8d0e-45bf-97c8-a047ed85c30c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13490" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2afdfc9fd54 namespace: kserve-ci-e2e-test resourceVersion: "13567" uid: 9e9026fc-0a0b-42bd-a5ac-5a4e416acd7d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2afe1197853 namespace: kserve-ci-e2e-test resourceVersion: "13569" uid: c3c47b23-69cc-4691-96e8-0ffe8aa7304d reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" in 12.679s (12.679s including waiting). Image size: 1334058092 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b2d4dc6dcb namespace: kserve-ci-e2e-test resourceVersion: "13744" uid: 77f61b37-b31b-4764-9629-932674643fac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b2d7c96930 namespace: kserve-ci-e2e-test resourceVersion: "13745" uid: 643dffe4-fe9f-43fd-a898-2fe6bb70e3b5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b2d83db706 namespace: kserve-ci-e2e-test resourceVersion: "13746" uid: 2bf346f6-a7b3-4cda-a3c9-6d682c669b34 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-04-22T18:55:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:26Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b2d84afa3c namespace: kserve-ci-e2e-test resourceVersion: "13747" uid: e20ef319-278a-45ee-811a-3c8f1c831edf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.31s (2.31s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b361fd3e21 namespace: kserve-ci-e2e-test resourceVersion: "13789" uid: f65a6a25-3dd5-47c8-b0e5-e8988ff5550d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b365139611 namespace: kserve-ci-e2e-test resourceVersion: "13793" uid: be21ac88-6c81-4946-84a3-c8eb03696551 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:55:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:55:28Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b36584c433 namespace: kserve-ci-e2e-test resourceVersion: "13794" uid: 383db08f-ce28-4660-b621-443054762801 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:55:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:56:06Z" message: 'Readiness probe failed: dial tcp 10.132.0.17:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T18:55:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:06Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2b3d9554340 namespace: kserve-ci-e2e-test resourceVersion: "14153" uid: 62ade7d5-b227-43d9-a804-1bf76abcc621 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2c5bdb4a809 namespace: kserve-ci-e2e-test resourceVersion: "14608" uid: 818fb937-1703-40ff-9a90-8be8648d7789 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql namespace: kserve-ci-e2e-test resourceVersion: "13480" uid: 4ef84b19-d69a-4d3e-8917-b80a93056b4b kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: success-200-isvc-9844b-predictor-758ffd9676-tf4ql.18a8c2c5bdb5adc0 namespace: kserve-ci-e2e-test resourceVersion: "14611" uid: 3f00dea0-be78-4a06-85c4-35e6547800dc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-9844b-predictor-758ffd9676 namespace: kserve-ci-e2e-test resourceVersion: "13463" uid: 9cdbd789-c0b8-41e5-9406-86b12cccf5a8 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'Created pod: success-200-isvc-9844b-predictor-758ffd9676-tf4ql' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b-predictor-758ffd9676.18a8c2afc69a1df4 namespace: kserve-ci-e2e-test resourceVersion: "13479" uid: cf6df345-84f8-434e-a2b9-a9f762e5160c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13462" uid: a415f9bb-f3d0-49b4-b77d-7a0da80b4399 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: Scaled up replica set success-200-isvc-9844b-predictor-758ffd9676 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b-predictor.18a8c2afbfa55c60 namespace: kserve-ci-e2e-test resourceVersion: "13464" uid: 2bd5708c-f677-4326-b577-86c4d4b344e2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13472" uid: fc71551e-8a63-4706-adc8-7b53605a0409 kind: Event lastTimestamp: "2026-04-22T18:56:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:13Z" name: success-200-isvc-9844b-predictor.18a8c2b344a3c62f namespace: kserve-ci-e2e-test resourceVersion: "14200" uid: ee15416c-0792-444e-b491-e755fcae6122 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13472" uid: fc71551e-8a63-4706-adc8-7b53605a0409 kind: Event lastTimestamp: "2026-04-22T18:56:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T18:55:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:13Z" name: success-200-isvc-9844b-predictor.18a8c2b344e3196f namespace: kserve-ci-e2e-test resourceVersion: "14203" uid: 01de7849-af6d-4ea3-bedc-b28226186244 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: fc71551e-8a63-4706-adc8-7b53605a0409 kind: Event lastTimestamp: "2026-04-22T18:56:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:43Z" name: success-200-isvc-9844b-predictor.18a8c2ba4296149d namespace: kserve-ci-e2e-test resourceVersion: "14500" uid: d0063ae1-ead6-4345-b9db-b269a14f54e2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T18:55:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-9844b-predictor namespace: kserve-ci-e2e-test resourceVersion: "13774" uid: fc71551e-8a63-4706-adc8-7b53605a0409 kind: Event lastTimestamp: "2026-04-22T18:56:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:55:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:43Z" name: success-200-isvc-9844b-predictor.18a8c2ba42f99d7c namespace: kserve-ci-e2e-test resourceVersion: "14501" uid: 282a6b52-f077-43cb-b708-91afbd985a76 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "13453" uid: ea503b0f-b0b9-4bca-b6ba-34c4f57c3481 kind: Event lastTimestamp: "2026-04-22T18:56:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:16Z" name: success-200-isvc-9844b.18a8c2afc5c515bf namespace: kserve-ci-e2e-test resourceVersion: "14255" uid: 9e2c023e-7851-4e48-b9cd-bc1de7851606 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "13453" uid: ea503b0f-b0b9-4bca-b6ba-34c4f57c3481 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'Failed to update status for InferenceService "success-200-isvc-9844b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9844b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b.18a8c2afc609bf31 namespace: kserve-ci-e2e-test resourceVersion: "13475" uid: 5578531c-47fb-476a-b881-bb96044b8143 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:55:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "13453" uid: ea503b0f-b0b9-4bca-b6ba-34c4f57c3481 kind: Event lastTimestamp: "2026-04-22T18:55:13Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-9844b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T18:55:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:55:13Z" name: success-200-isvc-9844b.18a8c2afc60a1179 namespace: kserve-ci-e2e-test resourceVersion: "13477" uid: 4f10f045-8f8e-4c01-b36d-eb50ccda8334 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-9844b namespace: kserve-ci-e2e-test resourceVersion: "14250" uid: ea503b0f-b0b9-4bca-b6ba-34c4f57c3481 kind: Event lastTimestamp: "2026-04-22T18:56:16Z" message: InferenceService [success-200-isvc-9844b] is Ready metadata: creationTimestamp: "2026-04-22T18:56:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:16Z" name: success-200-isvc-9844b.18a8c2be9383c58c namespace: kserve-ci-e2e-test resourceVersion: "14251" uid: b0c5eeb4-5516-44d6-b7bf-d8859d1da186 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:16:27.574956Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24285" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c8908-predictor-b7c844446-s6swb to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d87cfd1a72 namespace: kserve-ci-e2e-test resourceVersion: "24292" uid: a262b869-08a4-4e8a-94a7-ea4be5b5c52e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-c8908-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d889cd5673 namespace: kserve-ci-e2e-test resourceVersion: "24323" uid: 45135791-6ea0-4a8d-87fc-0518343705a7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24304" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8ba008ae2 namespace: kserve-ci-e2e-test resourceVersion: "24343" uid: d3310121-4e35-4eb8-9db4-22efe3377f53 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8bb48df93 namespace: kserve-ci-e2e-test resourceVersion: "24345" uid: bc24a8dd-988f-48c2-89e6-b5f6ce0a180a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8be4e5fd4 namespace: kserve-ci-e2e-test resourceVersion: "24346" uid: b1428247-1b0e-478e-936d-44c1234fcc22 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8bec75a86 namespace: kserve-ci-e2e-test resourceVersion: "24347" uid: 345dd2cc-7c3e-46bd-bde8-f2c47b0efe55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8bed3de88 namespace: kserve-ci-e2e-test resourceVersion: "24349" uid: b709bb7b-1915-4919-b55d-9f0f46ce8305 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8c27e1217 namespace: kserve-ci-e2e-test resourceVersion: "24350" uid: 9e2acfc4-8ac1-4f25-af74-4043b0a51275 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:16:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:16:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:28Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d8c2f90757 namespace: kserve-ci-e2e-test resourceVersion: "24351" uid: e8d92ad3-377c-4107-9f6f-c638dc06c8a2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:16:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:17:16Z" message: 'Readiness probe failed: dial tcp 10.132.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:16:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:16Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3d913ab0c6e namespace: kserve-ci-e2e-test resourceVersion: "24873" uid: 17304e08-5a75-4235-9e7e-03d94d3d0d0d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3eb2c7b5c1f namespace: kserve-ci-e2e-test resourceVersion: "25237" uid: fc7dfaf5-28a9-46d1-9d13-3f968c6e7791 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:17:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:47Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3eb2c7bfef1 namespace: kserve-ci-e2e-test resourceVersion: "25240" uid: e52c36db-671b-4c2c-a14b-04052fd93fa9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c8908-predictor-b7c844446-s6swb namespace: kserve-ci-e2e-test resourceVersion: "24289" uid: 577c7ee9-bb8b-48ec-854a-02e79872459b kind: Event lastTimestamp: "2026-04-22T19:17:51Z" message: 'Readiness probe failed: Get "https://10.132.0.25:8643/healthz": dial tcp 10.132.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:51Z" name: success-200-isvc-c8908-predictor-b7c844446-s6swb.18a8c3ebefe3d9c5 namespace: kserve-ci-e2e-test resourceVersion: "25354" uid: 2ed59ef8-8114-4963-9def-2a7e8bca2fe3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c8908-predictor-b7c844446 namespace: kserve-ci-e2e-test resourceVersion: "24261" uid: cb2e38ff-aaf1-4406-96f5-b4e90745f1f3 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'Created pod: success-200-isvc-c8908-predictor-b7c844446-s6swb' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908-predictor-b7c844446.18a8c3d87c9b7b70 namespace: kserve-ci-e2e-test resourceVersion: "24288" uid: b9f5e11a-cf65-4e8b-a86f-031b1eb90e01 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24257" uid: f9504354-8b98-41c9-8b89-e834c4444fad kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Scaled up replica set success-200-isvc-c8908-predictor-b7c844446 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908-predictor.18a8c3d878d31341 namespace: kserve-ci-e2e-test resourceVersion: "24263" uid: 98389be4-ad69-4d3e-8c97-763d35372394 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24259" uid: 78ec9bb3-11cd-49ac-a7c5-278dacca47b8 kind: Event lastTimestamp: "2026-04-22T19:17:12Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:12Z" name: success-200-isvc-c8908-predictor.18a8c3dbf749542b namespace: kserve-ci-e2e-test resourceVersion: "24845" uid: 5bf9071b-c320-46a3-9d0b-fd8f437125b1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:16:42Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24259" uid: 78ec9bb3-11cd-49ac-a7c5-278dacca47b8 kind: Event lastTimestamp: "2026-04-22T19:17:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:16:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:12Z" name: success-200-isvc-c8908-predictor.18a8c3dbf7a7263b namespace: kserve-ci-e2e-test resourceVersion: "24846" uid: b7e225f7-e4d9-42d3-8854-fdebc4caacad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24472" uid: 78ec9bb3-11cd-49ac-a7c5-278dacca47b8 kind: Event lastTimestamp: "2026-04-22T19:17:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:42Z" name: success-200-isvc-c8908-predictor.18a8c3df765466db namespace: kserve-ci-e2e-test resourceVersion: "25137" uid: e709ec02-298b-4ddb-85f1-7b8f8a480c82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:16:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c8908-predictor namespace: kserve-ci-e2e-test resourceVersion: "24472" uid: 78ec9bb3-11cd-49ac-a7c5-278dacca47b8 kind: Event lastTimestamp: "2026-04-22T19:17:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:16:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:42Z" name: success-200-isvc-c8908-predictor.18a8c3df76b25add namespace: kserve-ci-e2e-test resourceVersion: "25138" uid: 364d159a-ed83-4930-bf05-659556478b31 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: d51c0641-50fa-4696-ac8d-8a5a667eaf60 kind: Event lastTimestamp: "2026-04-22T19:17:26Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:26Z" name: success-200-isvc-c8908.18a8c3d878a7243c namespace: kserve-ci-e2e-test resourceVersion: "24977" uid: b0f90570-e69d-42b6-b280-809489ab0fb0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: d51c0641-50fa-4696-ac8d-8a5a667eaf60 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'Failed to update status for InferenceService "success-200-isvc-c8908": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c8908": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908.18a8c3d879053c1f namespace: kserve-ci-e2e-test resourceVersion: "24265" uid: df7a4630-ab95-4ccd-ad11-fd64d3e9284a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24236" uid: d51c0641-50fa-4696-ac8d-8a5a667eaf60 kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c8908": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-c8908.18a8c3d87905bd08 namespace: kserve-ci-e2e-test resourceVersion: "24266" uid: 30c50cb7-254b-4da5-a956-dae44d22716f reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c8908 namespace: kserve-ci-e2e-test resourceVersion: "24975" uid: d51c0641-50fa-4696-ac8d-8a5a667eaf60 kind: Event lastTimestamp: "2026-04-22T19:17:26Z" message: InferenceService [success-200-isvc-c8908] is Ready metadata: creationTimestamp: "2026-04-22T19:17:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:26Z" name: success-200-isvc-c8908.18a8c3e6221419b9 namespace: kserve-ci-e2e-test resourceVersion: "24976" uid: b7b0a25a-19f0-4029-ae6d-0671cbe8b25e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:17:03.441981Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24686" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0d6d447d9 namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: 1a8f1410-ee01-4f95-8fb1-37e7fc3306bf reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24702" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f002cbab namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: cf7544ee-2a70-45f1-931b-7889de1d2ff5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f138abf4 namespace: kserve-ci-e2e-test resourceVersion: "24747" uid: 5afb5795-9074-448a-99d7-2c5208e86b8f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f40164bd namespace: kserve-ci-e2e-test resourceVersion: "24751" uid: acdca87c-0532-4bc1-a3c1-7f2e10172a88 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f4a612d1 namespace: kserve-ci-e2e-test resourceVersion: "24752" uid: 301e5225-601d-41a9-8576-c767c30bffc8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f4b47c34 namespace: kserve-ci-e2e-test resourceVersion: "24753" uid: cfb1f811-4603-4b90-a25a-de5295709e9f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f78a6e2c namespace: kserve-ci-e2e-test resourceVersion: "24754" uid: 68a23501-8159-4e6e-bc33-d512498a0f29 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:04Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e0f807001a namespace: kserve-ci-e2e-test resourceVersion: "24755" uid: 1e332277-4ac5-4f9a-8cda-d53e1aa7b532 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-04-22T19:17:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:17:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:17:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:51Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c3e1403a4b00 namespace: kserve-ci-e2e-test resourceVersion: "25355" uid: 81188e2f-3610-4183-a0bd-36ad8deb764c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:34:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:33Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c4d53abb6a15 namespace: kserve-ci-e2e-test resourceVersion: "32761" uid: 31db1403-dd4d-41e3-82be-be61b0872d64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp namespace: kserve-ci-e2e-test resourceVersion: "24690" uid: fed8332d-991a-446a-a7f2-afb45300ed5f kind: Event lastTimestamp: "2026-04-22T19:34:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:34:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:33Z" name: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp.18a8c4d53abc194c namespace: kserve-ci-e2e-test resourceVersion: "32762" uid: 3925de4d-a1b7-4d61-8d97-1d8123b69fff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e7803-predictor-64dbd4f98d namespace: kserve-ci-e2e-test resourceVersion: "24671" uid: 12c98daf-e375-44d8-9ceb-6b3cecc9e7df kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Created pod: success-200-isvc-e7803-predictor-64dbd4f98d-vc5wp' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor-64dbd4f98d.18a8c3e0d66620db namespace: kserve-ci-e2e-test resourceVersion: "24689" uid: d9f686d8-da2e-444e-b414-5a125691f21b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24664" uid: 770887a6-940f-465c-96ed-257b3c53ba78 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Scaled up replica set success-200-isvc-e7803-predictor-64dbd4f98d from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803-predictor.18a8c3e0d48a8a47 namespace: kserve-ci-e2e-test resourceVersion: "24677" uid: bcb41247-893a-485c-886f-697863abb96d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24670" uid: fc4e121c-cbf3-4291-a120-24e51d15f322 kind: Event lastTimestamp: "2026-04-22T19:17:33Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:33Z" name: success-200-isvc-e7803-predictor.18a8c3e452f8d2d2 namespace: kserve-ci-e2e-test resourceVersion: "25032" uid: 92ff8713-e51f-4d7b-80e6-dbd69ec73fd3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24670" uid: fc4e121c-cbf3-4291-a120-24e51d15f322 kind: Event lastTimestamp: "2026-04-22T19:17:33Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:17:33Z" name: success-200-isvc-e7803-predictor.18a8c3e453554276 namespace: kserve-ci-e2e-test resourceVersion: "25033" uid: ec083987-153c-4154-8f85-254d4b74e13b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: fc4e121c-cbf3-4291-a120-24e51d15f322 kind: Event lastTimestamp: "2026-04-22T19:18:18Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:18Z" name: success-200-isvc-e7803-predictor.18a8c3eb50fe1d1f namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: 65fdb167-31ab-4b8c-a9e4-b9a7eeeca6b0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:17:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e7803-predictor namespace: kserve-ci-e2e-test resourceVersion: "24903" uid: fc4e121c-cbf3-4291-a120-24e51d15f322 kind: Event lastTimestamp: "2026-04-22T19:18:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:17:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:18Z" name: success-200-isvc-e7803-predictor.18a8c3eb515c973b namespace: kserve-ci-e2e-test resourceVersion: "25634" uid: 1bb492b1-186e-4732-a914-fd0b166b62fe reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 9f6e3399-ad6c-49ae-883e-6de199a821a6 kind: Event lastTimestamp: "2026-04-22T19:18:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:01Z" name: success-200-isvc-e7803.18a8c3e0d4666edb namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: 59c04983-1e8e-484b-9d30-ef448205c938 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 9f6e3399-ad6c-49ae-883e-6de199a821a6 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'Failed to update status for InferenceService "success-200-isvc-e7803": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e7803": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803.18a8c3e0d5170f66 namespace: kserve-ci-e2e-test resourceVersion: "24681" uid: 3936701e-6e25-47c1-b99c-d0c4bd9258b8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "24654" uid: 9f6e3399-ad6c-49ae-883e-6de199a821a6 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-e7803": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-e7803.18a8c3e0d51762a4 namespace: kserve-ci-e2e-test resourceVersion: "24684" uid: 807633c5-62ec-4181-9ac2-bcd53994676a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25451" uid: 9f6e3399-ad6c-49ae-883e-6de199a821a6 kind: Event lastTimestamp: "2026-04-22T19:18:01Z" message: InferenceService [success-200-isvc-e7803] is Ready metadata: creationTimestamp: "2026-04-22T19:18:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:01Z" name: success-200-isvc-e7803.18a8c3ee4e4741da namespace: kserve-ci-e2e-test resourceVersion: "25452" uid: 01d65b6e-a6d9-4ee0-906a-77990af29b80 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:12.764726Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19885" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f6fce-predictor-78694d6766-cnm2x to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3574fb8af64 namespace: kserve-ci-e2e-test resourceVersion: "19890" uid: 5da8cd62-b4c1-4e1b-a08b-4a64c9e17050 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f6fce-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c35760c7d87c namespace: kserve-ci-e2e-test resourceVersion: "19937" uid: ed92edd1-4e1e-41c7-b224-0476f8fca465 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19903" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3578c395232 namespace: kserve-ci-e2e-test resourceVersion: "19953" uid: 65fcb2ed-a51e-4162-85ac-390e80438161 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3578d7d8b7a namespace: kserve-ci-e2e-test resourceVersion: "19955" uid: 8d2335e1-8bcb-4072-9bdc-d10104352970 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c357904cea07 namespace: kserve-ci-e2e-test resourceVersion: "19956" uid: f43b8bab-d13a-4791-bebf-432c1d078757 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c35790b83b33 namespace: kserve-ci-e2e-test resourceVersion: "19957" uid: 080f2540-763d-4e26-97df-2bf9e953eb9d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c35790c45f83 namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: 1f58d035-9fa0-458f-8c61-e8efaf5e8052 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3579372a0f6 namespace: kserve-ci-e2e-test resourceVersion: "19959" uid: 5a65db16-0c42-4a67-8dc1-8ddabfb1ed81 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:13Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c35793de74a4 namespace: kserve-ci-e2e-test resourceVersion: "19960" uid: bca3d749-61e8-40c3-9107-4ae373b76fcb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:07:51Z" message: 'Readiness probe failed: dial tcp 10.132.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:51Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c357f8708868 namespace: kserve-ci-e2e-test resourceVersion: "20486" uid: be864871-6fa8-482e-86dc-3ee51d6b9948 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3d878190995 namespace: kserve-ci-e2e-test resourceVersion: "24258" uid: bd63bc8d-5a7c-46fc-a97a-291ee2aa9fc5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:16:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x namespace: kserve-ci-e2e-test resourceVersion: "19888" uid: f99c4911-3d98-40b7-82d2-f9a31411aa1c kind: Event lastTimestamp: "2026-04-22T19:16:27Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:16:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:16:27Z" name: success-200-isvc-f6fce-predictor-78694d6766-cnm2x.18a8c3d87819bb06 namespace: kserve-ci-e2e-test resourceVersion: "24260" uid: 9f94b0e6-a72a-4654-9b9c-5f3f1c41d585 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f6fce-predictor-78694d6766 namespace: kserve-ci-e2e-test resourceVersion: "19872" uid: a87a9887-abb5-41bd-99e1-bf1eb08973ba kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'Created pod: success-200-isvc-f6fce-predictor-78694d6766-cnm2x' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-f6fce-predictor-78694d6766.18a8c3574f578260 namespace: kserve-ci-e2e-test resourceVersion: "19886" uid: 1e5301d9-e05d-4457-94ae-ee478bab1e06 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: 17553a6b-9e24-4cc4-895f-5efc454eed47 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: Scaled up replica set success-200-isvc-f6fce-predictor-78694d6766 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-f6fce-predictor.18a8c3574d779ff5 namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: d3843880-4953-49c2-b9f4-b5ce35bcc78a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 84606fcb-d0f5-4d80-b3fb-a1af64ca055e kind: Event lastTimestamp: "2026-04-22T19:07:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:42Z" name: success-200-isvc-f6fce-predictor.18a8c35acbf491fc namespace: kserve-ci-e2e-test resourceVersion: "20273" uid: 10e26d09-54c1-49a2-a3e8-21d66b2f2c87 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:07:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "19871" uid: 84606fcb-d0f5-4d80-b3fb-a1af64ca055e kind: Event lastTimestamp: "2026-04-22T19:07:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:07:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:42Z" name: success-200-isvc-f6fce-predictor.18a8c35acc56a426 namespace: kserve-ci-e2e-test resourceVersion: "20274" uid: 1ac0dd76-54e6-4486-bdaf-f21b2c1faa83 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 84606fcb-d0f5-4d80-b3fb-a1af64ca055e kind: Event lastTimestamp: "2026-04-22T19:08:42Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:42Z" name: success-200-isvc-f6fce-predictor.18a8c361c9c9b26b namespace: kserve-ci-e2e-test resourceVersion: "20977" uid: 6da96d1a-ac05-4c5f-a333-a6bf0d8bc077 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-04-22T19:07:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f6fce-predictor namespace: kserve-ci-e2e-test resourceVersion: "20125" uid: 84606fcb-d0f5-4d80-b3fb-a1af64ca055e kind: Event lastTimestamp: "2026-04-22T19:08:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:07:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:42Z" name: success-200-isvc-f6fce-predictor.18a8c361ca243bb9 namespace: kserve-ci-e2e-test resourceVersion: "20978" uid: 113617c3-9092-4f2e-b5e1-8adecb1cf0b1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 8f8fea48-64c8-40bd-9de7-4242381ec8f7 kind: Event lastTimestamp: "2026-04-22T19:08:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:01Z" name: success-200-isvc-f6fce.18a8c3574d52c62b namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: e4364825-79dc-4993-8331-64077bdbc1b7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 8f8fea48-64c8-40bd-9de7-4242381ec8f7 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-f6fce": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6fce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-f6fce.18a8c3574dd34c5b namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: b7a5bca0-c2b4-47a7-8fa8-40a6cdede424 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "19851" uid: 8f8fea48-64c8-40bd-9de7-4242381ec8f7 kind: Event lastTimestamp: "2026-04-22T19:07:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-f6fce": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:12Z" name: success-200-isvc-f6fce.18a8c3574dd39eb8 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 8f106843-2bf1-4cc6-a7e5-23450df317e4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f6fce namespace: kserve-ci-e2e-test resourceVersion: "20584" uid: 8f8fea48-64c8-40bd-9de7-4242381ec8f7 kind: Event lastTimestamp: "2026-04-22T19:08:01Z" message: InferenceService [success-200-isvc-f6fce] is Ready metadata: creationTimestamp: "2026-04-22T19:08:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:01Z" name: success-200-isvc-f6fce.18a8c362b28be342 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 0287cc8f-93f9-4a44-aac7-673e8a76ed6a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:07:48.674586Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20382" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd to ip-10-0-138-84.ec2.internal metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35fac1da309 namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: a0f0ad5c-eea1-4cdd-8586-e03ac9438646 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ffb29-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35fb39acf5d namespace: kserve-ci-e2e-test resourceVersion: "20432" uid: 1665c28c-e03d-4500-97b5-da9c9dd9e74e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20397" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35fea37f114 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 2d07da7e-5dd7-411e-a571-fc048e4aff92 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1293" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35feb7f188b namespace: kserve-ci-e2e-test resourceVersion: "20450" uid: 391a58ca-4788-4247-b6b1-6f7532aca1bb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35fee64d60c namespace: kserve-ci-e2e-test resourceVersion: "20453" uid: b8d4e053-07b5-492b-967d-f7a3095a818c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35feed273a4 namespace: kserve-ci-e2e-test resourceVersion: "20455" uid: 5ca0ac14-7307-4910-be85-4d003970da42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35feeddd0bb namespace: kserve-ci-e2e-test resourceVersion: "20456" uid: 230ed376-1e7d-4500-89fd-0109e49dcb82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35ff16fecaf namespace: kserve-ci-e2e-test resourceVersion: "20460" uid: 35fb9675-0f39-499c-93b3-36a3e664b9a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:07:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:07:49Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c35ff1ff4f7f namespace: kserve-ci-e2e-test resourceVersion: "20461" uid: 1cdc77b7-45eb-4bc3-978e-296805982666 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:07:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:08:27Z" message: 'Readiness probe failed: dial tcp 10.132.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-04-22T19:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:08:27Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c360605bc9db namespace: kserve-ci-e2e-test resourceVersion: "20838" uid: a2f64063-77e2-4766-85d8-7b2c1b9c1e60 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c3e0d5be5ebd namespace: kserve-ci-e2e-test resourceVersion: "24685" uid: 3c657c1c-334e-4d23-be29-21a3b3936e32 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd namespace: kserve-ci-e2e-test resourceVersion: "20384" uid: 72e8ea5f-a383-4292-80ae-d6039c848008 kind: Event lastTimestamp: "2026-04-22T19:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-04-22T19:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:17:03Z" name: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd.18a8c3e0d5bee806 namespace: kserve-ci-e2e-test resourceVersion: "24688" uid: 50d19dd6-c8ab-4c4f-a578-120c0f8e3077 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-138-84.ec2.internal source: component: kubelet host: ip-10-0-138-84.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ffb29-predictor-b8d4948fb namespace: kserve-ci-e2e-test resourceVersion: "20365" uid: b900a061-cc97-43b2-a6d6-690d1e899e40 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'Created pod: success-200-isvc-ffb29-predictor-b8d4948fb-5wxhd' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29-predictor-b8d4948fb.18a8c35fab182e6e namespace: kserve-ci-e2e-test resourceVersion: "20383" uid: efcfbed3-4310-414d-9151-4994cca47cba reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20362" uid: 9fb83128-aa57-4690-8ad8-f927ab6d6a7b kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: Scaled up replica set success-200-isvc-ffb29-predictor-b8d4948fb from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29-predictor.18a8c35fa8cdb5a9 namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 576cb3e5-191e-479d-8dbb-4ddc41423665 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: db4f8f32-c521-494b-9ffd-1785e42747b5 kind: Event lastTimestamp: "2026-04-22T19:08:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:18Z" name: success-200-isvc-ffb29-predictor.18a8c36327af881e namespace: kserve-ci-e2e-test resourceVersion: "20769" uid: 49e39829-6023-4353-b8c5-d291cc38f798 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-04-22T19:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20364" uid: db4f8f32-c521-494b-9ffd-1785e42747b5 kind: Event lastTimestamp: "2026-04-22T19:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:08:18Z" name: success-200-isvc-ffb29-predictor.18a8c363280c945a namespace: kserve-ci-e2e-test resourceVersion: "20770" uid: 668470ca-c48d-4e95-89c8-0386d9580ef1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: db4f8f32-c521-494b-9ffd-1785e42747b5 kind: Event lastTimestamp: "2026-04-22T19:09:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:09:03Z" name: success-200-isvc-ffb29-predictor.18a8c36a25c3e39c namespace: kserve-ci-e2e-test resourceVersion: "21176" uid: 7fe57075-5a19-416e-8b41-81690867a2b7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ffb29-predictor namespace: kserve-ci-e2e-test resourceVersion: "20608" uid: db4f8f32-c521-494b-9ffd-1785e42747b5 kind: Event lastTimestamp: "2026-04-22T19:09:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T19:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:09:03Z" name: success-200-isvc-ffb29-predictor.18a8c36a2620a148 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: da364d9c-4a0d-4d1a-885c-5913c7752055 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 2be1fccf-0894-4984-852f-f64a49291db3 kind: Event lastTimestamp: "2026-04-22T19:08:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:37Z" name: success-200-isvc-ffb29.18a8c35fa8cd9ada namespace: kserve-ci-e2e-test resourceVersion: "20938" uid: ad815422-3508-4b08-9c34-4764020319f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 2be1fccf-0894-4984-852f-f64a49291db3 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'Failed to update status for InferenceService "success-200-isvc-ffb29": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ffb29": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29.18a8c35fa96b4d8e namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 5b2ed889-ef2d-4749-8599-8884a6280a2f reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20348" uid: 2be1fccf-0894-4984-852f-f64a49291db3 kind: Event lastTimestamp: "2026-04-22T19:07:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ffb29": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-04-22T19:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:07:48Z" name: success-200-isvc-ffb29.18a8c35fa96b9389 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 10f8c4c1-bc56-4dd8-8790-c0f236bd37cf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:08:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ffb29 namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: 2be1fccf-0894-4984-852f-f64a49291db3 kind: Event lastTimestamp: "2026-04-22T19:08:37Z" message: InferenceService [success-200-isvc-ffb29] is Ready metadata: creationTimestamp: "2026-04-22T19:08:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:08:37Z" name: success-200-isvc-ffb29.18a8c36b1a3f7379 namespace: kserve-ci-e2e-test resourceVersion: "20936" uid: 0fe09622-56c8-40c0-9301-d28617ad305b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:57:47.645970Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-70f3d-6496fb9568-5tskj to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:57:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:57:47Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c2d3bc04c649 namespace: kserve-ci-e2e-test resourceVersion: "15448" uid: fed9e2c9-c368-4e77-b650-a01eea8be55b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15452" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T18:57:48Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:57:48Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c2d3d54ad452 namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 19ca55b5-9a29-4546-a1ee-ede569fb094d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-70f3d} kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T18:57:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:48Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c2d3d6940107 namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: 6a307180-fbbf-428e-9ede-feecde11a2d9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-70f3d} kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T18:57:48Z" message: 'Created container: switch-graph-70f3d' metadata: creationTimestamp: "2026-04-22T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:48Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c2d3d99a42cc namespace: kserve-ci-e2e-test resourceVersion: "15470" uid: 78eb70d6-1587-4f5e-bf5f-3287d3b73742 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-70f3d} kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T18:57:48Z" message: Started container switch-graph-70f3d metadata: creationTimestamp: "2026-04-22T18:57:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:48Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c2d3da07ab7a namespace: kserve-ci-e2e-test resourceVersion: "15471" uid: bcd687fd-2d2b-4db8-b071-dc7969333a05 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:06:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-70f3d} kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T19:06:02Z" message: Stopping container switch-graph-70f3d metadata: creationTimestamp: "2026-04-22T19:06:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:02Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c346ebd2da13 namespace: kserve-ci-e2e-test resourceVersion: "18955" uid: 0727683a-a93f-4f31-a9fe-c45513d1a7e1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T19:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-70f3d} kind: Pod name: switch-graph-70f3d-6496fb9568-5tskj namespace: kserve-ci-e2e-test resourceVersion: "15443" uid: d659fae6-80e2-4e52-8641-98e62d0721f2 kind: Event lastTimestamp: "2026-04-22T19:06:29Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:06:29Z" name: switch-graph-70f3d-6496fb9568-5tskj.18a8c3475b98a3a1 namespace: kserve-ci-e2e-test resourceVersion: "19284" uid: f7da42ea-aabf-4aaa-bfbc-623e9d39107a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-70f3d-6496fb9568 namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: 22897543-e405-4210-a367-349fd201b05f kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: 'Created pod: switch-graph-70f3d-6496fb9568-5tskj' metadata: creationTimestamp: "2026-04-22T18:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: switch-graph-70f3d-6496fb9568.18a8c2d3bbb12aa5 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: cd180492-30da-4416-b746-21cce476b50a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15434" uid: 54c579ea-1b9f-410a-8c36-ca2cc9038ca9 kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: Scaled up replica set switch-graph-70f3d-6496fb9568 from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:57:47Z" name: switch-graph-70f3d.18a8c2d3bb033c6d namespace: kserve-ci-e2e-test resourceVersion: "15439" uid: 3ae5c66d-de39-4c0f-b985-ac26948d906d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:57:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15457" uid: 18572275-8169-4ceb-8116-7f64eb7f53c5 kind: Event lastTimestamp: "2026-04-22T18:57:47Z" message: InferenceGraph [switch-graph-70f3d] is Ready metadata: creationTimestamp: "2026-04-22T18:57:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:57:47Z" name: switch-graph-70f3d.18a8c2d3bfcd84ea namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 1b70c7ed-a089-4188-82dc-17f5073e1ba3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: c56e3485-263c-4137-80ef-b42edcd8a2f7 kind: Event lastTimestamp: "2026-04-22T18:58:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:02Z" name: switch-graph-70f3d.18a8c2d73958a11f namespace: kserve-ci-e2e-test resourceVersion: "15589" uid: 050d8e93-8d03-4646-a8e8-4c15bc43a0af reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:58:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-70f3d namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: c56e3485-263c-4137-80ef-b42edcd8a2f7 kind: Event lastTimestamp: "2026-04-22T18:58:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-04-22T18:58:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:58:02Z" name: switch-graph-70f3d.18a8c2d73994d092 namespace: kserve-ci-e2e-test resourceVersion: "15591" uid: 49b3f420-b5ba-407e-9dd2-a83815a11db8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-04-22T18:56:33.221207Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-9844b-8475b5564f-rnmkl to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T18:56:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T18:56:33Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c267f50897 namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: 29bf08e7-0b45-47be-9dc3-30047198e864 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:33Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-9844b-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T18:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:33Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c274c2e769 namespace: kserve-ci-e2e-test resourceVersion: "14417" uid: 5bd3db15-bec2-4d9b-b50b-fd3272df230e reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:34Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:34Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T18:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T18:56:34Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c2a5874bc2 namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: cfeff58b-8c55-499f-9b74-76a9bbc178cc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:34Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" metadata: creationTimestamp: "2026-04-22T18:56:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:34Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c2a703a8d9 namespace: kserve-ci-e2e-test resourceVersion: "14427" uid: c2c018ae-f40b-44ae-b57e-6ad3cb0bb66c reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:37Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" in 3.052s (3.052s including waiting). Image size: 216233553 bytes.' metadata: creationTimestamp: "2026-04-22T18:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:37Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c35cfb8493 namespace: kserve-ci-e2e-test resourceVersion: "14453" uid: 5c2c07c6-0f70-4a22-b8c2-628b0a0e0fcc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:37Z" message: 'Created container: switch-graph-9844b' metadata: creationTimestamp: "2026-04-22T18:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:37Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c3601a3924 namespace: kserve-ci-e2e-test resourceVersion: "14454" uid: 72c2a548-b08d-4d64-8936-25a57fcc476d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:37Z" message: Started container switch-graph-9844b metadata: creationTimestamp: "2026-04-22T18:56:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:37Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c36090f140 namespace: kserve-ci-e2e-test resourceVersion: "14455" uid: 67e96838-f4fd-48f8-b514-6337fd2e9f7b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:56:47Z" message: Stopping container switch-graph-9844b metadata: creationTimestamp: "2026-04-22T18:56:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:56:47Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c5b6ff0476 namespace: kserve-ci-e2e-test resourceVersion: "14579" uid: 420be9c4-ba59-4763-836b-cabb8eb08b79 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-04-22T18:56:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-9844b} kind: Pod name: switch-graph-9844b-8475b5564f-rnmkl namespace: kserve-ci-e2e-test resourceVersion: "14400" uid: d61a4cd2-d237-4f6a-b403-57164324e96c kind: Event lastTimestamp: "2026-04-22T18:57:13Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T18:56:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T18:57:13Z" name: switch-graph-9844b-8475b5564f-rnmkl.18a8c2c6136e682e namespace: kserve-ci-e2e-test resourceVersion: "14955" uid: 5f75f4f9-a16a-45c3-8d18-a1c176a3d05c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-9844b-8475b5564f namespace: kserve-ci-e2e-test resourceVersion: "14395" uid: 98128c66-abcf-477e-b0b7-74fd1c7abfe8 kind: Event lastTimestamp: "2026-04-22T18:56:33Z" message: 'Created pod: switch-graph-9844b-8475b5564f-rnmkl' metadata: creationTimestamp: "2026-04-22T18:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:33Z" name: switch-graph-9844b-8475b5564f.18a8c2c267991626 namespace: kserve-ci-e2e-test resourceVersion: "14401" uid: 3cb1d1eb-62fe-4e12-9f35-917a13419c2d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-9844b namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: 7a523d13-5cea-44fc-8064-9e477879111a kind: Event lastTimestamp: "2026-04-22T18:56:33Z" message: Scaled up replica set switch-graph-9844b-8475b5564f from 0 to 1 metadata: creationTimestamp: "2026-04-22T18:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T18:56:33Z" name: switch-graph-9844b.18a8c2c266b07349 namespace: kserve-ci-e2e-test resourceVersion: "14396" uid: 232e722d-4e6c-42ef-a087-0f42b2a99426 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T18:56:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-9844b namespace: kserve-ci-e2e-test resourceVersion: "14411" uid: 6b2fee78-ca39-4e82-bac1-673a717c9e4e kind: Event lastTimestamp: "2026-04-22T18:56:33Z" message: InferenceGraph [switch-graph-9844b] is Ready metadata: creationTimestamp: "2026-04-22T18:56:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T18:56:33Z" name: switch-graph-9844b.18a8c2c26b34b516 namespace: kserve-ci-e2e-test resourceVersion: "14413" uid: 5ce5c7b7-2026-4267-9ec4-602b3403125d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-04-22T19:18:13.509820Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25561" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-e7803-7c4596f696-44m5s to ip-10-0-143-56.ec2.internal metadata: creationTimestamp: "2026-04-22T19:18:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-04-22T19:18:13Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f12732a305 namespace: kserve-ci-e2e-test resourceVersion: "25566" uid: b9541c2d-17e9-45c6-bae8-007af997f088 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-6886c4dbcc-pjszt source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:13Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:18:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-e7803-serving-cert" not found' metadata: creationTimestamp: "2026-04-22T19:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:13Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f12fda65e6 namespace: kserve-ci-e2e-test resourceVersion: "25581" uid: aa602047-7cb6-412d-89c9-99afd0fd6c09 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:14Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25571" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:18:14Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-04-22T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-04-22T19:18:14Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f164803cdc namespace: kserve-ci-e2e-test resourceVersion: "25589" uid: 46657e71-9075-4c14-b572-a30a74b4287a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7803} kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:18:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:0169f8fa51f9c0b83c5c9a121f51af9f3e3cdcc48ff1b00c2fcaa12b38a7914f" already present on machine metadata: creationTimestamp: "2026-04-22T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:14Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f165c54b20 namespace: kserve-ci-e2e-test resourceVersion: "25591" uid: 6fe833fc-153d-42b9-bce1-b71dcaffecd7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7803} kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:18:14Z" message: 'Created container: switch-graph-e7803' metadata: creationTimestamp: "2026-04-22T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:14Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f16904a75f namespace: kserve-ci-e2e-test resourceVersion: "25592" uid: bff2f4aa-3bc7-4453-b00f-94e485ace6e9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7803} kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:18:14Z" message: Started container switch-graph-e7803 metadata: creationTimestamp: "2026-04-22T19:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:18:14Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c3f16975bdfb namespace: kserve-ci-e2e-test resourceVersion: "25593" uid: f94f5b03-cc3b-42b4-86f8-e4c2866de825 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:34:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7803} kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:34:32Z" message: Stopping container switch-graph-e7803 metadata: creationTimestamp: "2026-04-22T19:34:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:32Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c4d533266c61 namespace: kserve-ci-e2e-test resourceVersion: "32746" uid: 1fd96f28-bf9f-4af7-a472-07d26a67f7e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-04-22T19:34:36Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-e7803} kind: Pod name: switch-graph-e7803-7c4596f696-44m5s namespace: kserve-ci-e2e-test resourceVersion: "25563" uid: cf542cf9-fe06-43fb-bb3f-00c8830493b4 kind: Event lastTimestamp: "2026-04-22T19:34:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-04-22T19:34:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-04-22T19:34:56Z" name: switch-graph-e7803-7c4596f696-44m5s.18a8c4d606325d85 namespace: kserve-ci-e2e-test resourceVersion: "32986" uid: 956fc2b2-af47-47b2-99de-2693296cf410 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-56.ec2.internal source: component: kubelet host: ip-10-0-143-56.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-e7803-7c4596f696 namespace: kserve-ci-e2e-test resourceVersion: "25558" uid: 7041aec3-2cb6-47d5-a0f5-1b2e762d482e kind: Event lastTimestamp: "2026-04-22T19:18:13Z" message: 'Created pod: switch-graph-e7803-7c4596f696-44m5s' metadata: creationTimestamp: "2026-04-22T19:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:13Z" name: switch-graph-e7803-7c4596f696.18a8c3f126d4542e namespace: kserve-ci-e2e-test resourceVersion: "25562" uid: ed5302c6-2299-4b61-a6b2-2305f09a0080 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25556" uid: f078661e-ca38-453a-9c53-55f9df90b989 kind: Event lastTimestamp: "2026-04-22T19:18:13Z" message: Scaled up replica set switch-graph-e7803-7c4596f696 from 0 to 1 metadata: creationTimestamp: "2026-04-22T19:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:13Z" name: switch-graph-e7803.18a8c3f126125b77 namespace: kserve-ci-e2e-test resourceVersion: "25559" uid: 5b2ed0ab-8f54-432a-892c-6a15b49ac9d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-04-22T19:18:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25574" uid: 236120f1-b81c-4243-a05c-445a7fc4f5f9 kind: Event lastTimestamp: "2026-04-22T19:18:13Z" message: InferenceGraph [switch-graph-e7803] is Ready metadata: creationTimestamp: "2026-04-22T19:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-04-22T19:18:13Z" name: switch-graph-e7803.18a8c3f12abf5d8a namespace: kserve-ci-e2e-test resourceVersion: "25576" uid: f6250b84-1941-4301-bd4c-f02d209c28f8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: d6f5c15c-cfbf-4820-a307-355d23a6f070 kind: Event lastTimestamp: "2026-04-22T19:18:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:58Z" name: switch-graph-e7803.18a8c3f4a476580f namespace: kserve-ci-e2e-test resourceVersion: "25994" uid: c06eb395-78eb-4474-a97d-96bbf7ad337f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-04-22T19:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-e7803 namespace: kserve-ci-e2e-test resourceVersion: "25557" uid: d6f5c15c-cfbf-4820-a307-355d23a6f070 kind: Event lastTimestamp: "2026-04-22T19:18:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-04-22T19:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-04-22T19:18:58Z" name: switch-graph-e7803.18a8c3f4a4d440d0 namespace: kserve-ci-e2e-test resourceVersion: "25995" uid: 545dd00a-268c-4f9e-a2e4-c232346f5c70 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "32994"