--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:40:47.225406Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22082" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-601fd-8f5c764c5-5vlqq to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:40:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:40:47Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c321880734f namespace: kserve-ci-e2e-test resourceVersion: "22089" uid: babbce3c-66e6-4399-99f9-e0456f6afd9e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:47Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-601fd-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:47Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c3226c4f14b namespace: kserve-ci-e2e-test resourceVersion: "22099" uid: ed70894b-7e7c-41f9-b3cb-d61bb40f6414 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:48Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22090" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:48Z" message: Add eth0 [10.132.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:40:48Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c325de76eca namespace: kserve-ci-e2e-test resourceVersion: "22109" uid: 41594dab-dac4-4628-ad36-7eeb90fcec2d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-601fd} kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:48Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c325fa93600 namespace: kserve-ci-e2e-test resourceVersion: "22111" uid: 38d1c7cf-607c-4961-b047-39d6f3a4664f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-601fd} kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:48Z" message: 'Created container: ensemble-graph-601fd' metadata: creationTimestamp: "2026-09-08T13:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:48Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c32626bd381 namespace: kserve-ci-e2e-test resourceVersion: "22112" uid: 0610dc50-5a8f-4775-9f40-02dfcc61a520 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-601fd} kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:48Z" message: Started container ensemble-graph-601fd metadata: creationTimestamp: "2026-09-08T13:40:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:48Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c3262e31ef2 namespace: kserve-ci-e2e-test resourceVersion: "22113" uid: 48232240-c187-49dd-b5c0-7d4edd0d2ac8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-601fd} kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Stopping container ensemble-graph-601fd metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c346e145ebe namespace: kserve-ci-e2e-test resourceVersion: "22207" uid: 9db6d4ae-8c14-40d3-a1aa-4cea3941e802 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:41:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-601fd} kind: Pod name: ensemble-graph-601fd-8f5c764c5-5vlqq namespace: kserve-ci-e2e-test resourceVersion: "22083" uid: fdcc6425-d6ad-40ab-9fe1-4e3c1ada15f0 kind: Event lastTimestamp: "2026-09-08T13:41:25Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:41:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:25Z" name: ensemble-graph-601fd-8f5c764c5-5vlqq.18d35c351fda4863 namespace: kserve-ci-e2e-test resourceVersion: "22615" uid: 17a1acb1-2ec0-463a-8436-5216b2e749c9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-601fd-8f5c764c5 namespace: kserve-ci-e2e-test resourceVersion: "22078" uid: caf9dc62-6589-448b-8672-d1a95a40900f kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'Created pod: ensemble-graph-601fd-8f5c764c5-5vlqq' metadata: creationTimestamp: "2026-09-08T13:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: ensemble-graph-601fd-8f5c764c5.18d35c3217c3d18c namespace: kserve-ci-e2e-test resourceVersion: "22086" uid: 833014f1-4d9b-4d5b-ae64-f4e11de37cd4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-601fd namespace: kserve-ci-e2e-test resourceVersion: "22075" uid: 61c1d919-6c6a-4517-93e6-399837a8ed3c kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: Scaled up replica set ensemble-graph-601fd-8f5c764c5 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: ensemble-graph-601fd.18d35c321649ca07 namespace: kserve-ci-e2e-test resourceVersion: "22081" uid: 36766a12-3231-4c31-8a6d-13a9f222ef0d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:47Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-601fd namespace: kserve-ci-e2e-test resourceVersion: "22093" uid: fe36cb12-28db-4964-ac07-3cbb68f2e88d kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: InferenceGraph [ensemble-graph-601fd] is Ready metadata: creationTimestamp: "2026-09-08T13:40:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:40:47Z" name: ensemble-graph-601fd.18d35c322065dc90 namespace: kserve-ci-e2e-test resourceVersion: "22095" uid: b8caeb7d-cf8f-4beb-a42d-4680bc6d900f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:41:57.493417Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23043" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-64898-76dcdb6787-r4g2b to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:41:57Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c4274cd3cab namespace: kserve-ci-e2e-test resourceVersion: "23048" uid: 78e58ae5-c75d-4591-a134-c25b5c8c11f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:41:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-64898-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:57Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c427f3ccc39 namespace: kserve-ci-e2e-test resourceVersion: "23064" uid: 826b0617-741b-4b91-a725-f5326dc1607a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:58Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23052" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:41:58Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:41:58Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c42ba4e2a2d namespace: kserve-ci-e2e-test resourceVersion: "23070" uid: a68cb77a-f666-47be-a1fc-706071643648 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-64898} kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:41:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:58Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c42bbae4c40 namespace: kserve-ci-e2e-test resourceVersion: "23072" uid: 9d355d24-f977-4bda-a752-61e8d974bd99 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-64898} kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:41:58Z" message: 'Created container: ensemble-graph-64898' metadata: creationTimestamp: "2026-09-08T13:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:58Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c42be8bd6d0 namespace: kserve-ci-e2e-test resourceVersion: "23075" uid: 01257c8a-28b7-45dd-beb3-7750ebf65447 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-64898} kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:41:58Z" message: Started container ensemble-graph-64898 metadata: creationTimestamp: "2026-09-08T13:41:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:58Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35c42bef8f0e0 namespace: kserve-ci-e2e-test resourceVersion: "23076" uid: eb3edec0-ab95-4ad1-a594-6a74704b5c18 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-64898} kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Stopping container ensemble-graph-64898 metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:12Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35cb5a0f2bb3d namespace: kserve-ci-e2e-test resourceVersion: "26842" uid: f838101c-18f6-4c18-adb9-5f00a9177e3d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-64898} kind: Pod name: ensemble-graph-64898-76dcdb6787-r4g2b namespace: kserve-ci-e2e-test resourceVersion: "23044" uid: ea6d2e82-76f9-4b33-8a9e-981f78846597 kind: Event lastTimestamp: "2026-09-08T13:50:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:40Z" name: ensemble-graph-64898-76dcdb6787-r4g2b.18d35cb667704629 namespace: kserve-ci-e2e-test resourceVersion: "27194" uid: 29a12013-a4ad-4725-8109-6d641252f66b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-64898-76dcdb6787 namespace: kserve-ci-e2e-test resourceVersion: "23040" uid: a52ff330-3639-49ba-9390-796d64ca1f6d kind: Event lastTimestamp: "2026-09-08T13:41:57Z" message: 'Created pod: ensemble-graph-64898-76dcdb6787-r4g2b' metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:57Z" name: ensemble-graph-64898-76dcdb6787.18d35c4274863d02 namespace: kserve-ci-e2e-test resourceVersion: "23046" uid: cd0311b7-7194-4ae0-b271-d2a9008d9ef8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-64898 namespace: kserve-ci-e2e-test resourceVersion: "23038" uid: ad238213-2670-4f65-a65d-fcd268e73708 kind: Event lastTimestamp: "2026-09-08T13:41:57Z" message: Scaled up replica set ensemble-graph-64898-76dcdb6787 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:57Z" name: ensemble-graph-64898.18d35c4273bc571e namespace: kserve-ci-e2e-test resourceVersion: "23041" uid: 0ef43ec2-9b9e-4e3c-83da-06d66f05e719 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-64898 namespace: kserve-ci-e2e-test resourceVersion: "23057" uid: 3131f1b2-a71a-457d-9627-3ecc89c20169 kind: Event lastTimestamp: "2026-09-08T13:41:57Z" message: InferenceGraph [ensemble-graph-64898] is Ready metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:57Z" name: ensemble-graph-64898.18d35c427920a434 namespace: kserve-ci-e2e-test resourceVersion: "23059" uid: 47331fb8-904b-4d51-9dde-0a4b60c57ee4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-64898 namespace: kserve-ci-e2e-test resourceVersion: "23039" uid: 70b61fd1-7d30-4594-bb65-2c428d0333b1 kind: Event lastTimestamp: "2026-09-08T13:42:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:27Z" name: ensemble-graph-64898.18d35c45f241de97 namespace: kserve-ci-e2e-test resourceVersion: "23323" uid: 95b9ac33-93c7-410c-a386-a75573ebd782 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:42:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-64898 namespace: kserve-ci-e2e-test resourceVersion: "23039" uid: 70b61fd1-7d30-4594-bb65-2c428d0333b1 kind: Event lastTimestamp: "2026-09-08T13:42:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:27Z" name: ensemble-graph-64898.18d35c45f2ae4a13 namespace: kserve-ci-e2e-test resourceVersion: "23324" uid: c28779de-725a-4be3-8e45-9f677a5c96bc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:50:12.513205Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-13b40-predictor-5cf559cf78-8pgvv to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5b647d2f9 namespace: kserve-ci-e2e-test resourceVersion: "26930" uid: 9085898b-f4b8-416f-b49b-f8c58e572491 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26936" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5d8cad295 namespace: kserve-ci-e2e-test resourceVersion: "26954" uid: 05de640a-69cb-4291-b993-1838031830bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5da4054dd namespace: kserve-ci-e2e-test resourceVersion: "26957" uid: 4232bc21-4859-4644-bb80-e87b178a3992 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5dd251fe9 namespace: kserve-ci-e2e-test resourceVersion: "26960" uid: dcfd2b87-933f-4db5-b00b-06adef721cc3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5dd9752ea namespace: kserve-ci-e2e-test resourceVersion: "26961" uid: 1154722a-0f89-4cc6-9f15-b9afb9cf149c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5dda20285 namespace: kserve-ci-e2e-test resourceVersion: "26963" uid: 13be1fac-4c26-4d32-9270-4e3a4cc83e13 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5e0cf1be4 namespace: kserve-ci-e2e-test resourceVersion: "26964" uid: 423312fe-9f17-4839-9e9f-a7bc1a6f76b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb5e13edcf2 namespace: kserve-ci-e2e-test resourceVersion: "26965" uid: 77c98ab7-6663-4e9c-8a7a-d63c0dacaf2d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:50:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:50Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cb636b78f60 namespace: kserve-ci-e2e-test resourceVersion: "27409" uid: 49d2cb14-e5b2-44fe-a30c-24d68c118650 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:22Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cc60ce7630c namespace: kserve-ci-e2e-test resourceVersion: "27787" uid: 7790cafa-1bf4-4758-a0ce-dc72eef613b6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:22Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cc60ce7cffc namespace: kserve-ci-e2e-test resourceVersion: "27789" uid: 824e8119-1837-40f2-80b1-cef010bf0674 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv namespace: kserve-ci-e2e-test resourceVersion: "26928" uid: 62f4f732-d73b-45e1-83f9-6a05cf2ed953 kind: Event lastTimestamp: "2026-09-08T13:51:25Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:25Z" name: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv.18d35cc6bf1b403c namespace: kserve-ci-e2e-test resourceVersion: "27898" uid: 0c0e9538-29b5-441e-91cd-17b45c96aa84 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-13b40-predictor-5cf559cf78 namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: b1e0c12c-4e16-4172-b858-f6d400525c79 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'Created pod: error-404-isvc-13b40-predictor-5cf559cf78-8pgvv' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-13b40-predictor-5cf559cf78.18d35cb5b5b15c13 namespace: kserve-ci-e2e-test resourceVersion: "26927" uid: 81fa6553-674f-4169-b5f3-4211f82692f3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26913" uid: 0351b002-919e-4ad9-a8b9-8a0b2bcad87e kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Scaled up replica set error-404-isvc-13b40-predictor-5cf559cf78 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-13b40-predictor.18d35cb5b3971c72 namespace: kserve-ci-e2e-test resourceVersion: "26919" uid: 084583ff-206e-4112-a19e-cfc1b0e8ee45 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 6e5c7b15-d316-46e7-aeb8-56b8db53048f kind: Event lastTimestamp: "2026-09-08T13:50:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:42Z" name: error-404-isvc-13b40-predictor.18d35cb93238c216 namespace: kserve-ci-e2e-test resourceVersion: "27210" uid: 2420582b-61f2-40d7-b1f0-db92442609c9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 6e5c7b15-d316-46e7-aeb8-56b8db53048f kind: Event lastTimestamp: "2026-09-08T13:50:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:42Z" name: error-404-isvc-13b40-predictor.18d35cb93291fe0b namespace: kserve-ci-e2e-test resourceVersion: "27211" uid: 1760e8db-7a13-4909-8b32-c5ac0bbc3457 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 6e5c7b15-d316-46e7-aeb8-56b8db53048f kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: error-404-isvc-13b40-predictor.18d35cc03045f18c namespace: kserve-ci-e2e-test resourceVersion: "27634" uid: 4209c254-ae85-49a3-a755-b2df5017b348 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "27090" uid: 6e5c7b15-d316-46e7-aeb8-56b8db53048f kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: error-404-isvc-13b40-predictor.18d35cc030b8b14b namespace: kserve-ci-e2e-test resourceVersion: "27637" uid: b33a4ba9-ae16-409a-87fa-dc4d72f6e65d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: e369fece-b4e2-4f04-95fe-9bfd6d9a7dbf kind: Event lastTimestamp: "2026-09-08T13:51:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:00Z" name: error-404-isvc-13b40.18d35cb5b3937395 namespace: kserve-ci-e2e-test resourceVersion: "27524" uid: 46914d0f-3d56-4b11-876d-25d01051bdc2 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: e369fece-b4e2-4f04-95fe-9bfd6d9a7dbf kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'Failed to update status for InferenceService "error-404-isvc-13b40": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-13b40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-13b40.18d35cb5b45ce48c namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 7f148c01-1cf8-4708-b89c-28c0bd43de8c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: e369fece-b4e2-4f04-95fe-9bfd6d9a7dbf kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-13b40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-13b40.18d35cb5b45d3e3c namespace: kserve-ci-e2e-test resourceVersion: "26924" uid: b16b82ff-0170-4b09-8183-6ffa3738a176 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "27522" uid: e369fece-b4e2-4f04-95fe-9bfd6d9a7dbf kind: Event lastTimestamp: "2026-09-08T13:51:00Z" message: InferenceService [error-404-isvc-13b40] is Ready metadata: creationTimestamp: "2026-09-08T13:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:00Z" name: error-404-isvc-13b40.18d35cc0f45953bc namespace: kserve-ci-e2e-test resourceVersion: "27523" uid: 4f3a8482-32eb-4f64-8e75-73c7a791d9ec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:31:08.216876Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17527" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:31:08Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab48e785a4 namespace: kserve-ci-e2e-test resourceVersion: "17531" uid: 0b9893c8-ea57-439f-8f4b-442a346461e5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-30f77-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:08Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab591a212f namespace: kserve-ci-e2e-test resourceVersion: "17548" uid: 602825d8-2439-4f51-a82b-1cc0b48af743 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17536" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab8eb3512e namespace: kserve-ci-e2e-test resourceVersion: "17563" uid: 7fe532f7-0587-475b-87e9-3b0271394c09 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab90014c4d namespace: kserve-ci-e2e-test resourceVersion: "17565" uid: 4c69465a-50c5-4ec5-ac1f-de2cba2068b8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab92cee1df namespace: kserve-ci-e2e-test resourceVersion: "17568" uid: a92bb4f3-4caf-456d-8f08-93b7f7c8ab24 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab935d8c79 namespace: kserve-ci-e2e-test resourceVersion: "17569" uid: c9d45c4d-abbf-4396-99d3-f25e7a9b271f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab93690369 namespace: kserve-ci-e2e-test resourceVersion: "17570" uid: 46cb67d1-a955-40ae-8f08-e02f560b81a5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab967f0adf namespace: kserve-ci-e2e-test resourceVersion: "17572" uid: 3c37e87c-9a6c-46b6-b462-aa08f779874d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35bab96f103cb namespace: kserve-ci-e2e-test resourceVersion: "17573" uid: 2e6e70ae-4ea6-48b7-b18c-e797edd9c3be reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:31:47Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:47Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35babfd240507 namespace: kserve-ci-e2e-test resourceVersion: "17959" uid: 4458d7e0-799d-4fd6-9e8c-98faa2108e34 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:22Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35c2c6d81dc9d namespace: kserve-ci-e2e-test resourceVersion: "21764" uid: 986b7799-581f-4f45-b3d0-f97aa76ce011 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w namespace: kserve-ci-e2e-test resourceVersion: "17529" uid: d3ca8249-fe84-412a-9361-49044d970e51 kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:22Z" name: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w.18d35c2c6d827997 namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 3d2b19e3-1508-43d0-8b05-c97d12766e3e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-30f77-predictor-6c9bbd784d namespace: kserve-ci-e2e-test resourceVersion: "17521" uid: fc15bd80-ff2c-4064-803d-e867dee33e07 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: 'Created pod: error-404-isvc-30f77-predictor-6c9bbd784d-pwj9w' metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:08Z" name: error-404-isvc-30f77-predictor-6c9bbd784d.18d35bab48628034 namespace: kserve-ci-e2e-test resourceVersion: "17528" uid: bdc60e20-c1b9-4dfe-8f9c-afe692738597 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17519" uid: 3daa209e-1d5a-48f5-8976-bddbf9e4db2c kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: Scaled up replica set error-404-isvc-30f77-predictor-6c9bbd784d from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:08Z" name: error-404-isvc-30f77-predictor.18d35bab46a334f1 namespace: kserve-ci-e2e-test resourceVersion: "17522" uid: 45cd407a-8342-413e-900c-ff8471b9e674 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: e29ad654-c5de-4646-ba8a-70664a374755 kind: Event lastTimestamp: "2026-09-08T13:31:53Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:53Z" name: error-404-isvc-30f77-predictor.18d35baec523af49 namespace: kserve-ci-e2e-test resourceVersion: "18006" uid: 493bb50f-dd59-42cd-9a82-13eb0a514902 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17520" uid: e29ad654-c5de-4646-ba8a-70664a374755 kind: Event lastTimestamp: "2026-09-08T13:31:53Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:53Z" name: error-404-isvc-30f77-predictor.18d35baec59d886b namespace: kserve-ci-e2e-test resourceVersion: "18007" uid: e288f91b-46a4-4654-bac5-0a644196397e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: e29ad654-c5de-4646-ba8a-70664a374755 kind: Event lastTimestamp: "2026-09-08T13:32:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:23Z" name: error-404-isvc-30f77-predictor.18d35bb244523385 namespace: kserve-ci-e2e-test resourceVersion: "18288" uid: 518ccce4-80be-482f-8d16-dc2421fc6db4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17726" uid: e29ad654-c5de-4646-ba8a-70664a374755 kind: Event lastTimestamp: "2026-09-08T13:32:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:23Z" name: error-404-isvc-30f77-predictor.18d35bb244c2bf97 namespace: kserve-ci-e2e-test resourceVersion: "18289" uid: ce6ac629-1ee9-4625-9650-9dc37fdb9274 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30f77 namespace: kserve-ci-e2e-test resourceVersion: "17509" uid: 90acdd29-403e-4811-a069-9245b9229362 kind: Event lastTimestamp: "2026-09-08T13:31:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:57Z" name: error-404-isvc-30f77.18d35bab46a9a591 namespace: kserve-ci-e2e-test resourceVersion: "18057" uid: 79c9696d-41b4-445f-a2bb-941d9b33a788 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18055" uid: 90acdd29-403e-4811-a069-9245b9229362 kind: Event lastTimestamp: "2026-09-08T13:31:57Z" message: InferenceService [error-404-isvc-30f77] is Ready metadata: creationTimestamp: "2026-09-08T13:31:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:57Z" name: error-404-isvc-30f77.18d35bb6b8743cc4 namespace: kserve-ci-e2e-test resourceVersion: "18056" uid: d5e6dc6e-4f07-46ee-84fd-cb1ac6ea8468 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:39:47.341925Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-601fd-predictor-5978f84496-s6xpw to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c24272b217e namespace: kserve-ci-e2e-test resourceVersion: "21380" uid: d4b5b355-214b-4b90-a280-b6f35d9a1066 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c24496e8958 namespace: kserve-ci-e2e-test resourceVersion: "21416" uid: 853290ef-c790-41d8-989b-9c1455a29cfa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c244af5fe72 namespace: kserve-ci-e2e-test resourceVersion: "21418" uid: 290774eb-9d87-4530-b24a-a5e609494f5f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c244db181d9 namespace: kserve-ci-e2e-test resourceVersion: "21419" uid: 0edcc760-7333-4b34-8665-bcbb4b1e4145 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c244e22c887 namespace: kserve-ci-e2e-test resourceVersion: "21420" uid: 315f794b-2715-4b46-a1e6-bf338af72584 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c244e2bb671 namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: ed98938e-ea9d-4b4c-afc1-5d796ad90a5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c2450d22317 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: 356670d9-6b7a-4ad8-b927-346f8623b75e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c2451432d12 namespace: kserve-ci-e2e-test resourceVersion: "21423" uid: 9abb831f-a83a-450d-9500-d1403235f5f9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c24b1b30207 namespace: kserve-ci-e2e-test resourceVersion: "21865" uid: c49802ba-d8b6-4d61-ad80-e4558ba2eb52 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c347798dda8 namespace: kserve-ci-e2e-test resourceVersion: "22243" uid: dedc1f23-4958-4740-9f26-879c023ed329 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-601fd-predictor-5978f84496-s6xpw namespace: kserve-ci-e2e-test resourceVersion: "21377" uid: f547ade2-8456-48a3-bc4e-668072492c63 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: error-404-isvc-601fd-predictor-5978f84496-s6xpw.18d35c3477998679 namespace: kserve-ci-e2e-test resourceVersion: "22244" uid: d249c87c-c64b-4f43-b24d-0d6201c9d238 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-601fd-predictor-5978f84496 namespace: kserve-ci-e2e-test resourceVersion: "21369" uid: ded3a3be-5edd-4b48-a88e-3a162c0dc76d kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: 'Created pod: error-404-isvc-601fd-predictor-5978f84496-s6xpw' metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor-5978f84496.18d35c2426d15bd0 namespace: kserve-ci-e2e-test resourceVersion: "21378" uid: b3475bb4-ec26-4ccc-b669-97ec4274b4fd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21367" uid: 58901afc-b986-4b10-82b8-5bceaadb64d4 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Scaled up replica set error-404-isvc-601fd-predictor-5978f84496 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-601fd-predictor.18d35c2423c9ab7c namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: b64dce32-74de-410b-9392-364a02863d67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 2c462e65-b780-437a-b480-0646ce03600f kind: Event lastTimestamp: "2026-09-08T13:40:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:17Z" name: error-404-isvc-601fd-predictor.18d35c27a30a596f namespace: kserve-ci-e2e-test resourceVersion: "21670" uid: d4fd0d75-8a4c-4e71-91bc-ea65fdf1fdca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 2c462e65-b780-437a-b480-0646ce03600f kind: Event lastTimestamp: "2026-09-08T13:40:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:17Z" name: error-404-isvc-601fd-predictor.18d35c27a393d1d9 namespace: kserve-ci-e2e-test resourceVersion: "21671" uid: 89ebd2b4-90b5-4cb1-92d3-c9b89bad00fa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 2c462e65-b780-437a-b480-0646ce03600f kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: error-404-isvc-601fd-predictor.18d35c2ea14bd123 namespace: kserve-ci-e2e-test resourceVersion: "22096" uid: 561a63ec-6822-46a0-a46f-b7921e3ab772 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 2c462e65-b780-437a-b480-0646ce03600f kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: error-404-isvc-601fd-predictor.18d35c2ea1cf4488 namespace: kserve-ci-e2e-test resourceVersion: "22097" uid: da719dfb-aa6a-49be-9991-a2c7e3424b6d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-601fd namespace: kserve-ci-e2e-test resourceVersion: "21352" uid: 6c92993c-2517-45cf-8b06-a2f1d2d6bcb1 kind: Event lastTimestamp: "2026-09-08T13:40:35Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:40:35Z" name: error-404-isvc-601fd.18d35c2423e0722c namespace: kserve-ci-e2e-test resourceVersion: "21971" uid: 00eb59af-6491-4846-9281-c38c1373b1c7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-601fd namespace: kserve-ci-e2e-test resourceVersion: "21969" uid: 6c92993c-2517-45cf-8b06-a2f1d2d6bcb1 kind: Event lastTimestamp: "2026-09-08T13:40:35Z" message: InferenceService [error-404-isvc-601fd] is Ready metadata: creationTimestamp: "2026-09-08T13:40:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:40:35Z" name: error-404-isvc-601fd.18d35c2f6df6fe96 namespace: kserve-ci-e2e-test resourceVersion: "21970" uid: c3720b43-62c4-4a7a-beb8-64bc98153baf reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:40:57.613806Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22285" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-64898-predictor-c55974db4-6czmd to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:40:57Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c3483b2f081 namespace: kserve-ci-e2e-test resourceVersion: "22289" uid: 95c5ab8e-77e9-41e6-ad12-32f2b8e29e4e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34a5b17a1a namespace: kserve-ci-e2e-test resourceVersion: "22310" uid: 5b13d46c-f0b0-433d-a750-4106bfa13dc7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34a7ad9fda namespace: kserve-ci-e2e-test resourceVersion: "22312" uid: 4b4f8d41-06a4-4332-8210-c00e5d9c1b69 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34aa70ade3 namespace: kserve-ci-e2e-test resourceVersion: "22313" uid: 4d1f42b9-bb3a-4f8f-a10a-f7915ad6370b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34aadeb025 namespace: kserve-ci-e2e-test resourceVersion: "22314" uid: 74b33acd-a833-4271-b5cd-c16fc567dbfd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34aae9265a namespace: kserve-ci-e2e-test resourceVersion: "22315" uid: 4cb5e98c-bbe1-4d5e-82d4-be2a7630bf2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34ae06bed4 namespace: kserve-ci-e2e-test resourceVersion: "22317" uid: b0ddfa2f-ba75-4c44-812c-2a9d9a22909b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34ae72d097 namespace: kserve-ci-e2e-test resourceVersion: "22318" uid: 1d04ecd2-324d-41d9-9662-753051621fad reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-08T13:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:50:15Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:15Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35c34d72d94ac namespace: kserve-ci-e2e-test resourceVersion: "26985" uid: 51f07735-58d7-4870-9ad3-be72cdae7d0a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35cb5b3640b3f namespace: kserve-ci-e2e-test resourceVersion: "26917" uid: 512a4c96-da85-4dfd-926f-17f6ba1596e3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:12Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35cb5b3649cc8 namespace: kserve-ci-e2e-test resourceVersion: "26920" uid: 48a90a89-156e-44ba-b42a-b0718bd9658c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-64898-predictor-c55974db4-6czmd namespace: kserve-ci-e2e-test resourceVersion: "22287" uid: c00540df-0c64-499b-87b3-e89c1afabef2 kind: Event lastTimestamp: "2026-09-08T13:50:15Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:15Z" name: error-404-isvc-64898-predictor-c55974db4-6czmd.18d35cb64ba58547 namespace: kserve-ci-e2e-test resourceVersion: "26984" uid: b00ab008-65b5-40d6-8cdd-5bf1c15d8b5a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-64898-predictor-c55974db4 namespace: kserve-ci-e2e-test resourceVersion: "22279" uid: 67ba0a6f-bb59-44ef-8e45-ec2832c47c46 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: 'Created pod: error-404-isvc-64898-predictor-c55974db4-6czmd' metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:57Z" name: error-404-isvc-64898-predictor-c55974db4.18d35c34834293c6 namespace: kserve-ci-e2e-test resourceVersion: "22286" uid: 189352b6-7009-4ea1-a68d-1d91f3c017d5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22276" uid: a7de4fdf-984a-4017-9777-67ccb422fef9 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Scaled up replica set error-404-isvc-64898-predictor-c55974db4 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:57Z" name: error-404-isvc-64898-predictor.18d35c3480ff17ed namespace: kserve-ci-e2e-test resourceVersion: "22281" uid: 292be07d-aa8e-4cdf-8c26-bfc48afd250e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: 20edd7e4-f733-4cb2-938b-3e1a6b950245 kind: Event lastTimestamp: "2026-09-08T13:41:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:42Z" name: error-404-isvc-64898-predictor.18d35c37ff852194 namespace: kserve-ci-e2e-test resourceVersion: "22895" uid: a119ea8b-2474-4b5b-ac03-4efd19453de6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22278" uid: 20edd7e4-f733-4cb2-938b-3e1a6b950245 kind: Event lastTimestamp: "2026-09-08T13:41:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:42Z" name: error-404-isvc-64898-predictor.18d35c38000e9ce3 namespace: kserve-ci-e2e-test resourceVersion: "22896" uid: 7bbbabba-5d8f-4079-a458-8db93488e080 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: 20edd7e4-f733-4cb2-938b-3e1a6b950245 kind: Event lastTimestamp: "2026-09-08T13:42:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:12Z" name: error-404-isvc-64898-predictor.18d35c427c4b6784 namespace: kserve-ci-e2e-test resourceVersion: "23195" uid: f92de8c3-c510-4a2c-8c2b-c9044d7b6b3d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:41:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22470" uid: 20edd7e4-f733-4cb2-938b-3e1a6b950245 kind: Event lastTimestamp: "2026-09-08T13:42:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:12Z" name: error-404-isvc-64898-predictor.18d35c427c936f9e namespace: kserve-ci-e2e-test resourceVersion: "23196" uid: 3a16013e-d70f-4b34-ac4d-9e2981c9f745 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64898 namespace: kserve-ci-e2e-test resourceVersion: "22261" uid: ef0e3d68-f44d-4fce-8a37-970b3e64c41a kind: Event lastTimestamp: "2026-09-08T13:41:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:45Z" name: error-404-isvc-64898.18d35c34810e8668 namespace: kserve-ci-e2e-test resourceVersion: "22923" uid: 721aae41-089a-4cd7-ad32-5dfdd9b83d61 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-64898 namespace: kserve-ci-e2e-test resourceVersion: "22921" uid: ef0e3d68-f44d-4fce-8a37-970b3e64c41a kind: Event lastTimestamp: "2026-09-08T13:41:45Z" message: InferenceService [error-404-isvc-64898] is Ready metadata: creationTimestamp: "2026-09-08T13:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:45Z" name: error-404-isvc-64898.18d35c3f92d0721e namespace: kserve-ci-e2e-test resourceVersion: "22922" uid: eed65d1b-b55d-4baf-937b-7d8c0f929b65 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:51:22.945969Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27827" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7d014-predictor-57c444947d-llcg9 to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:51:22Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc61c666caa namespace: kserve-ci-e2e-test resourceVersion: "27831" uid: 09b88704-ac6b-465c-937d-1ef6085bb181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7d014-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:23Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc62ae847ba namespace: kserve-ci-e2e-test resourceVersion: "27845" uid: 0ce57fce-1a8d-408e-b188-f6568883cee4 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27837" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc6621ab043 namespace: kserve-ci-e2e-test resourceVersion: "27866" uid: 756d7c7b-2050-430f-95de-8849afb1ad32 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc6638297c2 namespace: kserve-ci-e2e-test resourceVersion: "27870" uid: bc335a6a-6ca1-4e5e-8cda-53c0e4e8973f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc666a43224 namespace: kserve-ci-e2e-test resourceVersion: "27871" uid: e0fe356d-f7be-4ca2-82a5-85f61253a138 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc667130955 namespace: kserve-ci-e2e-test resourceVersion: "27872" uid: b85dd758-300b-41bb-a4d3-2551e33e60f2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc6671cea4c namespace: kserve-ci-e2e-test resourceVersion: "27873" uid: 38120a4c-1bbf-4c76-816e-51b4cc245b31 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc66a1c7805 namespace: kserve-ci-e2e-test resourceVersion: "27877" uid: e60c1028-0a30-4041-897b-ed5166902a05 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc66ab5a9a8 namespace: kserve-ci-e2e-test resourceVersion: "27878" uid: dbed8fd8-3272-4534-ac23-9046bbc01cd1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T13:52:01Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:52:01Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35cc6988bc568 namespace: kserve-ci-e2e-test resourceVersion: "28241" uid: 2d673d91-1d6a-4b90-be84-66bc337d6967 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T14:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T14:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:00:37Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35d47400db736 namespace: kserve-ci-e2e-test resourceVersion: "31924" uid: b41f8f52-7c72-4510-8345-cc8df63d26be reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7d014-predictor-57c444947d-llcg9 namespace: kserve-ci-e2e-test resourceVersion: "27829" uid: 842e9175-e850-428d-a685-db646f801d9c kind: Event lastTimestamp: "2026-09-08T14:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T14:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:00:37Z" name: error-404-isvc-7d014-predictor-57c444947d-llcg9.18d35d47400e5c3b namespace: kserve-ci-e2e-test resourceVersion: "31925" uid: 9b75c1eb-ba49-4787-a7f8-7bc250842ff7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7d014-predictor-57c444947d namespace: kserve-ci-e2e-test resourceVersion: "27821" uid: a33e7e8a-ca4d-4b52-b878-e46be729d767 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: 'Created pod: error-404-isvc-7d014-predictor-57c444947d-llcg9' metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:22Z" name: error-404-isvc-7d014-predictor-57c444947d.18d35cc61ba17396 namespace: kserve-ci-e2e-test resourceVersion: "27828" uid: 1f77ed00-cad5-45ec-b0c9-1aecef370f4f reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27819" uid: 5cf1c83e-41f0-44d0-8df0-3b738b347bbe kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Scaled up replica set error-404-isvc-7d014-predictor-57c444947d from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:22Z" name: error-404-isvc-7d014-predictor.18d35cc6196a205b namespace: kserve-ci-e2e-test resourceVersion: "27823" uid: 683090f2-509b-44c8-b1e7-2d5dbe33cf44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: c5523d04-2432-4976-80a0-b6bab7c60502 kind: Event lastTimestamp: "2026-09-08T13:52:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:07Z" name: error-404-isvc-7d014-predictor.18d35cc997e07328 namespace: kserve-ci-e2e-test resourceVersion: "28297" uid: d3be0f6f-7547-4da7-ad39-094a00b32be0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27820" uid: c5523d04-2432-4976-80a0-b6bab7c60502 kind: Event lastTimestamp: "2026-09-08T13:52:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:07Z" name: error-404-isvc-7d014-predictor.18d35cc99850607d namespace: kserve-ci-e2e-test resourceVersion: "28298" uid: bec8910b-4322-4ac7-ae98-2dac92fc07d0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "28021" uid: c5523d04-2432-4976-80a0-b6bab7c60502 kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: error-404-isvc-7d014-predictor.18d35ccd17113152 namespace: kserve-ci-e2e-test resourceVersion: "28575" uid: 16f22f7b-be6e-49a3-9ba1-497dbc654120 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:51:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "28021" uid: c5523d04-2432-4976-80a0-b6bab7c60502 kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: error-404-isvc-7d014-predictor.18d35ccd1767406c namespace: kserve-ci-e2e-test resourceVersion: "28577" uid: 8dac2234-0451-431d-8f85-9ccd5fb29503 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d014 namespace: kserve-ci-e2e-test resourceVersion: "27805" uid: 8b487902-7f5b-409b-a080-1ca2ea4045b5 kind: Event lastTimestamp: "2026-09-08T13:52:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:52:11Z" name: error-404-isvc-7d014.18d35cc61967241a namespace: kserve-ci-e2e-test resourceVersion: "28329" uid: bb0feaf4-77c1-46ed-9731-f930f4c4d602 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28327" uid: 8b487902-7f5b-409b-a080-1ca2ea4045b5 kind: Event lastTimestamp: "2026-09-08T13:52:11Z" message: InferenceService [error-404-isvc-7d014] is Ready metadata: creationTimestamp: "2026-09-08T13:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:52:11Z" name: error-404-isvc-7d014.18d35cd155247712 namespace: kserve-ci-e2e-test resourceVersion: "28328" uid: 3afeb577-64a0-4eee-a33e-0fb5ca2bdb75 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:30:32.240941Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17005" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-87be3-predictor-5656fc75d4-x59s5 to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba2e8924d3b namespace: kserve-ci-e2e-test resourceVersion: "17009" uid: 9a16bf3d-21ce-4390-b823-5d040897d43f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17015" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba30a6a1407 namespace: kserve-ci-e2e-test resourceVersion: "17029" uid: 81ca65e2-471c-4598-8e79-c77db596e097 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba30be514f2 namespace: kserve-ci-e2e-test resourceVersion: "17033" uid: 3f7f1f59-3672-45e3-9639-502e111af660 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba30fc83bcd namespace: kserve-ci-e2e-test resourceVersion: "17035" uid: a82dc50f-8830-4627-abca-55147f36a398 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba3103563dd namespace: kserve-ci-e2e-test resourceVersion: "17036" uid: 6276e8fc-8cba-4cfd-b899-9743458c6845 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba310402d09 namespace: kserve-ci-e2e-test resourceVersion: "17037" uid: 50271770-9a8c-43be-846d-ade9884b1e57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba312e9b480 namespace: kserve-ci-e2e-test resourceVersion: "17039" uid: 9380e597-70c1-48ca-ae46-232ecbeca6c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba3136a260b namespace: kserve-ci-e2e-test resourceVersion: "17040" uid: 31be1f9a-57fd-47de-97ee-f125701b0525 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:39:50Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:50Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35ba355285ff8 namespace: kserve-ci-e2e-test resourceVersion: "21451" uid: 9624c4b0-2295-480f-8807-590dd6730ea9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35c242ad4db57 namespace: kserve-ci-e2e-test resourceVersion: "21397" uid: 4881c7eb-7188-4438-af48-83a90568040c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35c242ad550c2 namespace: kserve-ci-e2e-test resourceVersion: "21398" uid: ddf776fb-6267-49cb-a949-3504edfb094c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5 namespace: kserve-ci-e2e-test resourceVersion: "17007" uid: d1821e85-bea3-485a-a68e-68953c5cae01 kind: Event lastTimestamp: "2026-09-08T13:39:50Z" message: 'Readiness probe failed: Get "https://10.134.0.36:8643/healthz": dial tcp 10.134.0.36:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:50Z" name: error-404-isvc-87be3-predictor-5656fc75d4-x59s5.18d35c24c99eb76f namespace: kserve-ci-e2e-test resourceVersion: "21450" uid: fc50ec92-bfa6-4ffa-a3d5-807ee1034deb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-87be3-predictor-5656fc75d4 namespace: kserve-ci-e2e-test resourceVersion: "16998" uid: 6d8a682e-91dc-469a-98ba-017cd9eb2fab kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'Created pod: error-404-isvc-87be3-predictor-5656fc75d4-x59s5' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor-5656fc75d4.18d35ba2e80585b1 namespace: kserve-ci-e2e-test resourceVersion: "17006" uid: ddc7c06d-5a11-4eb7-a0a2-8225322ee7c0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16994" uid: 1910dab8-f7c2-44c0-95f1-7a3a9871213b kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Scaled up replica set error-404-isvc-87be3-predictor-5656fc75d4 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-87be3-predictor.18d35ba2e67588ee namespace: kserve-ci-e2e-test resourceVersion: "17000" uid: 2bccaa7f-5791-48fd-a896-b8c6a57ce4ac reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16996" uid: 33d9316e-0b4c-4bf8-86c5-34c50300b2dd kind: Event lastTimestamp: "2026-09-08T13:31:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:02Z" name: error-404-isvc-87be3-predictor.18d35ba66527441f namespace: kserve-ci-e2e-test resourceVersion: "17370" uid: 69c8b665-7c5f-41c5-9f30-a3c04b41c012 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16996" uid: 33d9316e-0b4c-4bf8-86c5-34c50300b2dd kind: Event lastTimestamp: "2026-09-08T13:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:02Z" name: error-404-isvc-87be3-predictor.18d35ba665c6cc39 namespace: kserve-ci-e2e-test resourceVersion: "17371" uid: a77f05f7-a328-4678-bf84-aaf720eb78ed reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 33d9316e-0b4c-4bf8-86c5-34c50300b2dd kind: Event lastTimestamp: "2026-09-08T13:31:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:47Z" name: error-404-isvc-87be3-predictor.18d35bad6322d6c4 namespace: kserve-ci-e2e-test resourceVersion: "17960" uid: 480c1562-83a0-4fdb-a1bb-27e609a9a3ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17212" uid: 33d9316e-0b4c-4bf8-86c5-34c50300b2dd kind: Event lastTimestamp: "2026-09-08T13:31:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:47Z" name: error-404-isvc-87be3-predictor.18d35bad63896528 namespace: kserve-ci-e2e-test resourceVersion: "17961" uid: 9a056142-d8d1-4ddc-a6ff-444176ce969a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "16967" uid: 5d7f278a-9c4c-4ea9-876b-0403d2d880c1 kind: Event lastTimestamp: "2026-09-08T13:31:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:20Z" name: error-404-isvc-87be3.18d35ba2e6711d52 namespace: kserve-ci-e2e-test resourceVersion: "17689" uid: 560db569-f6e3-4a27-8bb8-1c3175c176b9 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17685" uid: 5d7f278a-9c4c-4ea9-876b-0403d2d880c1 kind: Event lastTimestamp: "2026-09-08T13:31:20Z" message: InferenceService [error-404-isvc-87be3] is Ready metadata: creationTimestamp: "2026-09-08T13:31:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:20Z" name: error-404-isvc-87be3.18d35bae106a9c15 namespace: kserve-ci-e2e-test resourceVersion: "17686" uid: db501e69-ef92-4fc7-affe-1a830fa7e917 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:41:33.490060Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22774" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c7da0-predictor-5488f9f968-qtw88 to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3cde1718a2 namespace: kserve-ci-e2e-test resourceVersion: "22778" uid: c85ab224-b090-40ec-be48-89aa60dd01f5 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-c7da0-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3ce98712d2 namespace: kserve-ci-e2e-test resourceVersion: "22788" uid: 9abc66fb-97d5-4402-a797-d22bd43098d5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22782" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d23b488b4 namespace: kserve-ci-e2e-test resourceVersion: "22806" uid: 887e2203-8ae4-4bed-9505-eda39fcc39e8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d2696fcdc namespace: kserve-ci-e2e-test resourceVersion: "22809" uid: b03906fb-9d69-49be-a8c3-757b97c42e45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d29c0a98f namespace: kserve-ci-e2e-test resourceVersion: "22811" uid: 77401c83-0cc9-489f-9236-9a52c5212aa3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d2a31d91c namespace: kserve-ci-e2e-test resourceVersion: "22812" uid: ec9ad6a9-8981-4dfe-8977-9f6c18ed4681 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d2a3d47e1 namespace: kserve-ci-e2e-test resourceVersion: "22813" uid: 638c0e72-c35b-454b-b921-8767ee7e2420 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d2d4ca673 namespace: kserve-ci-e2e-test resourceVersion: "22815" uid: 4da4e737-e7d1-478f-bf0d-0c88f8832764 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d2dc29188 namespace: kserve-ci-e2e-test resourceVersion: "22816" uid: 90b6a030-9336-46c5-920f-2b460ffc682e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:42:11Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:11Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35c3d4410f616 namespace: kserve-ci-e2e-test resourceVersion: "23181" uid: 63316f67-1bad-4339-81d0-97b9c28a6768 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:47Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35cbdf679fc29 namespace: kserve-ci-e2e-test resourceVersion: "27326" uid: ddf3fdf6-b9b4-4c19-a9d9-7ec41cf2b84e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88 namespace: kserve-ci-e2e-test resourceVersion: "22776" uid: 3303adaa-a8b2-46a5-ab7d-c7af4710ee7f kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:47Z" name: error-404-isvc-c7da0-predictor-5488f9f968-qtw88.18d35cbdf67aa16f namespace: kserve-ci-e2e-test resourceVersion: "27328" uid: 51c367ae-2dcb-43fb-adfb-ee63ea516b5a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c7da0-predictor-5488f9f968 namespace: kserve-ci-e2e-test resourceVersion: "22766" uid: 6c28662c-4443-465e-81d7-c524d2bce729 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: 'Created pod: error-404-isvc-c7da0-predictor-5488f9f968-qtw88' metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-c7da0-predictor-5488f9f968.18d35c3cdccd8ecf namespace: kserve-ci-e2e-test resourceVersion: "22775" uid: 2e864e84-e144-4c80-a424-77001c9f6f77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22764" uid: 981ba5d0-0dd2-4887-8f29-7d9a26309c33 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Scaled up replica set error-404-isvc-c7da0-predictor-5488f9f968 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-c7da0-predictor.18d35c3cd9267fa1 namespace: kserve-ci-e2e-test resourceVersion: "22768" uid: 78237533-a316-444c-898c-c0ec3859f959 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22765" uid: 7b2bda25-c325-4240-ac18-8e1cc55efc4a kind: Event lastTimestamp: "2026-09-08T13:41:48Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:48Z" name: error-404-isvc-c7da0-predictor.18d35c405777f557 namespace: kserve-ci-e2e-test resourceVersion: "22962" uid: 35e9ad98-95f4-419d-85ab-f950fc3dd21e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22765" uid: 7b2bda25-c325-4240-ac18-8e1cc55efc4a kind: Event lastTimestamp: "2026-09-08T13:41:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:48Z" name: error-404-isvc-c7da0-predictor.18d35c4057e47881 namespace: kserve-ci-e2e-test resourceVersion: "22963" uid: 398375b5-f5ae-4c9e-a04f-7b87b3fa80ea reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 7b2bda25-c325-4240-ac18-8e1cc55efc4a kind: Event lastTimestamp: "2026-09-08T13:42:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:48Z" name: error-404-isvc-c7da0-predictor.18d35c43d6a0393e namespace: kserve-ci-e2e-test resourceVersion: "23521" uid: c17cad9a-50ce-44ac-90b7-28ed47c77b2a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:42:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22961" uid: 7b2bda25-c325-4240-ac18-8e1cc55efc4a kind: Event lastTimestamp: "2026-09-08T13:42:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:48Z" name: error-404-isvc-c7da0-predictor.18d35c43d7112592 namespace: kserve-ci-e2e-test resourceVersion: "23523" uid: 311b70ea-33e0-455e-907d-591fb7b62dd6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "22750" uid: 9bb22e1c-e3b2-4c89-99d1-2946ca8b4b3d kind: Event lastTimestamp: "2026-09-08T13:42:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:42:21Z" name: error-404-isvc-c7da0.18d35c3cd923a696 namespace: kserve-ci-e2e-test resourceVersion: "23286" uid: 9713a55e-1371-41be-b118-954f9de3aa58 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23283" uid: 9bb22e1c-e3b2-4c89-99d1-2946ca8b4b3d kind: Event lastTimestamp: "2026-09-08T13:42:21Z" message: InferenceService [error-404-isvc-c7da0] is Ready metadata: creationTimestamp: "2026-09-08T13:42:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:42:21Z" name: error-404-isvc-c7da0.18d35c480123f858 namespace: kserve-ci-e2e-test resourceVersion: "23284" uid: a61fe6ee-34d4-4c6f-b0e8-b47e1893cc7d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:50:48.098472Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27344" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbdff53e585 namespace: kserve-ci-e2e-test resourceVersion: "27350" uid: 1be5156c-974a-4ce7-b6c1-8783a3c516d8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-ce1f6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe0a650636 namespace: kserve-ci-e2e-test resourceVersion: "27363" uid: ff7d5e0c-b3ed-4ec2-bd3f-6e9b17e4824b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27355" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe453d66ad namespace: kserve-ci-e2e-test resourceVersion: "27386" uid: 9a8fb810-7d26-468f-874b-f384b3099520 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe46fd46b5 namespace: kserve-ci-e2e-test resourceVersion: "27388" uid: ae61af3a-e9c6-49d6-9c95-d990d8fd2ce4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe49edc4b4 namespace: kserve-ci-e2e-test resourceVersion: "27389" uid: 32c730b2-e740-4e2c-8dfe-d5d4bea1e21a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe4a626c8f namespace: kserve-ci-e2e-test resourceVersion: "27390" uid: 233113e0-0dca-4a61-88fd-28d0ae84f98a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe4a6dfaa9 namespace: kserve-ci-e2e-test resourceVersion: "27392" uid: 8e47798f-74ee-48f5-9a94-b80ebc78ae52 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe4d61d7ec namespace: kserve-ci-e2e-test resourceVersion: "27393" uid: 121c47f1-84f1-4202-ac76-a46686b00765 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe4def97e4 namespace: kserve-ci-e2e-test resourceVersion: "27394" uid: 8fb39f03-4661-45fc-9ae4-f77ce9fbc514 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T13:51:25Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:25Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35cbe680bde0d namespace: kserve-ci-e2e-test resourceVersion: "27900" uid: 52a8abb9-4500-432c-b9ec-c0849d4e461e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T14:08:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T14:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:07Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35db001209f80 namespace: kserve-ci-e2e-test resourceVersion: "35036" uid: 2d8ba55b-bbf5-4f34-bf7c-5ed266f80f90 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk namespace: kserve-ci-e2e-test resourceVersion: "27348" uid: 425c75dd-b702-487e-b465-bb5e6709f43c kind: Event lastTimestamp: "2026-09-08T14:08:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T14:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:07Z" name: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk.18d35db0012103ad namespace: kserve-ci-e2e-test resourceVersion: "35037" uid: 10cc6fd5-df2e-4169-bee0-1322c330c5cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-ce1f6-predictor-b77d84c64 namespace: kserve-ci-e2e-test resourceVersion: "27338" uid: ed102a0a-2bcd-4c3b-a4ac-60e46afc6562 kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: 'Created pod: error-404-isvc-ce1f6-predictor-b77d84c64-v2lwk' metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6-predictor-b77d84c64.18d35cbdfe5665d4 namespace: kserve-ci-e2e-test resourceVersion: "27346" uid: 93fa25e9-b55e-4852-b185-cd1f82b47233 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27336" uid: f983dc99-cfd1-4aa7-a3c9-7f569da2963d kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: Scaled up replica set error-404-isvc-ce1f6-predictor-b77d84c64 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6-predictor.18d35cbdfb4bc0e5 namespace: kserve-ci-e2e-test resourceVersion: "27339" uid: 362793c8-a2d1-4d72-9823-726e1fd06b0d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27337" uid: 02ed2e23-346e-40a3-9b71-7e9c998ba100 kind: Event lastTimestamp: "2026-09-08T13:51:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:03Z" name: error-404-isvc-ce1f6-predictor.18d35cc179c90b43 namespace: kserve-ci-e2e-test resourceVersion: "27540" uid: 96926b14-2336-476c-8ddb-5263942d3669 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27337" uid: 02ed2e23-346e-40a3-9b71-7e9c998ba100 kind: Event lastTimestamp: "2026-09-08T13:51:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:03Z" name: error-404-isvc-ce1f6-predictor.18d35cc17a3eec12 namespace: kserve-ci-e2e-test resourceVersion: "27542" uid: a673522d-4d31-4d6c-b672-8f0479f8a058 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:51:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27541" uid: 02ed2e23-346e-40a3-9b71-7e9c998ba100 kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:48Z" name: error-404-isvc-ce1f6-predictor.18d35cc4f9381c57 namespace: kserve-ci-e2e-test resourceVersion: "28125" uid: db53aa27-fe93-476a-82e1-7e78a5e63473 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:51:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27541" uid: 02ed2e23-346e-40a3-9b71-7e9c998ba100 kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:48Z" name: error-404-isvc-ce1f6-predictor.18d35cc4f9ae7e38 namespace: kserve-ci-e2e-test resourceVersion: "28126" uid: 491aebb2-15c6-46d6-9883-d04b34a6b4c9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: a17e29ce-572f-4e37-b396-243ec395e76f kind: Event lastTimestamp: "2026-09-08T13:51:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:36Z" name: error-404-isvc-ce1f6.18d35cbdfb4ae02f namespace: kserve-ci-e2e-test resourceVersion: "27994" uid: 52198513-2350-4783-9c89-a9b0a2aeeb3e reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: a17e29ce-572f-4e37-b396-243ec395e76f kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: 'Failed to update status for InferenceService "error-404-isvc-ce1f6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce1f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6.18d35cbdfc1441ee namespace: kserve-ci-e2e-test resourceVersion: "27342" uid: aae6804c-6586-4f7c-8d77-de63ebe6b80c reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27324" uid: a17e29ce-572f-4e37-b396-243ec395e76f kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-ce1f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:48Z" name: error-404-isvc-ce1f6.18d35cbdfc149e93 namespace: kserve-ci-e2e-test resourceVersion: "27343" uid: 3ad240d9-fd7f-4dc4-a7ae-c2d8eeb0a8eb reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:35Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27992" uid: a17e29ce-572f-4e37-b396-243ec395e76f kind: Event lastTimestamp: "2026-09-08T13:51:35Z" message: InferenceService [error-404-isvc-ce1f6] is Ready metadata: creationTimestamp: "2026-09-08T13:51:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:35Z" name: error-404-isvc-ce1f6.18d35cc9247316c4 namespace: kserve-ci-e2e-test resourceVersion: "27993" uid: 6c023276-632d-49da-a10f-01d042416dfa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:40:23.207084Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21804" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e90df-predictor-5b7bc4c674-96vrn to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2c80e5fac2 namespace: kserve-ci-e2e-test resourceVersion: "21808" uid: dec519e5-c4bd-4bf6-aca8-e25a304b6999 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21812" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ca31e4d2c namespace: kserve-ci-e2e-test resourceVersion: "21824" uid: ab059f13-6f88-4091-9b25-33dd7da6abbf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ca4fe3757 namespace: kserve-ci-e2e-test resourceVersion: "21826" uid: 6f7ce2b7-879d-46ba-b99e-d7ba46a7a946 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ca7cfd6b1 namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: 5a36e661-8f81-412a-bb8b-4801751f04d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ca8447960 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: ff5d8614-8962-40b4-9636-a0a99cbf85a5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ca84e0382 namespace: kserve-ci-e2e-test resourceVersion: "21829" uid: 5f276476-4542-4db6-bd3f-9e0c2da7b470 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2cab22a6fd namespace: kserve-ci-e2e-test resourceVersion: "21832" uid: fdc3ce8d-e6be-4253-a442-03d11f3b2d91 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2cab915bdc namespace: kserve-ci-e2e-test resourceVersion: "21834" uid: ab6da1dd-52ab-43b2-a1c7-f3b673774cfe reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:41:00Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:00Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c2ce203c449 namespace: kserve-ci-e2e-test resourceVersion: "22359" uid: ee2387b8-1fbb-4467-a2df-a3326ac7e3f2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c3ccc7b81ab namespace: kserve-ci-e2e-test resourceVersion: "22732" uid: 637f0660-08a4-4a25-8f13-352752139adf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c3ccc7beec0 namespace: kserve-ci-e2e-test resourceVersion: "22733" uid: 7a68e4a2-9a43-42c1-ad7a-985c0fedb5ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn namespace: kserve-ci-e2e-test resourceVersion: "21807" uid: 181ccc0c-a0af-4812-9e89-60dce95568e6 kind: Event lastTimestamp: "2026-09-08T13:41:35Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:35Z" name: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn.18d35c3d6a2f6c4d namespace: kserve-ci-e2e-test resourceVersion: "22835" uid: 7118fa97-b844-41da-9a30-4711df9e3ac9 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e90df-predictor-5b7bc4c674 namespace: kserve-ci-e2e-test resourceVersion: "21798" uid: cca0c992-c9d0-4824-833f-9c4557e24364 kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: 'Created pod: error-404-isvc-e90df-predictor-5b7bc4c674-96vrn' metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor-5b7bc4c674.18d35c2c7fff3ebe namespace: kserve-ci-e2e-test resourceVersion: "21805" uid: b3771df1-8a07-4b69-ab97-6c1b896d67a2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21796" uid: 8ac6eae9-41ee-4ba6-a83f-561fea593d7d kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: Scaled up replica set error-404-isvc-e90df-predictor-5b7bc4c674 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:23Z" name: error-404-isvc-e90df-predictor.18d35c2c7d6f7472 namespace: kserve-ci-e2e-test resourceVersion: "21801" uid: 7d1860c5-6ed3-4091-b6df-77cd8d53efe6 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:40:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 5cfe2fde-ac05-4018-9650-2a65de98b26b kind: Event lastTimestamp: "2026-09-08T13:41:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:08Z" name: error-404-isvc-e90df-predictor.18d35c2ffbce47d6 namespace: kserve-ci-e2e-test resourceVersion: "22417" uid: a337a01c-8543-439b-afc7-89adab7843d1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:40:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21797" uid: 5cfe2fde-ac05-4018-9650-2a65de98b26b kind: Event lastTimestamp: "2026-09-08T13:41:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:08Z" name: error-404-isvc-e90df-predictor.18d35c2ffc326122 namespace: kserve-ci-e2e-test resourceVersion: "22418" uid: 8e9db9df-e735-47cf-b771-2050cdb184bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: 5cfe2fde-ac05-4018-9650-2a65de98b26b kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:23Z" name: error-404-isvc-e90df-predictor.18d35c3a78e7607b namespace: kserve-ci-e2e-test resourceVersion: "22589" uid: 0f70d35f-568a-4ca9-9e8c-3bfe3b7a412e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21999" uid: 5cfe2fde-ac05-4018-9650-2a65de98b26b kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:23Z" name: error-404-isvc-e90df-predictor.18d35c3a794d46a8 namespace: kserve-ci-e2e-test resourceVersion: "22591" uid: d113f34c-8031-4240-af5a-e8ec790b3b56 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e90df namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: d015d876-1fc8-4f65-9fc3-5d08253cf634 kind: Event lastTimestamp: "2026-09-08T13:41:10Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:10Z" name: error-404-isvc-e90df.18d35c2c7d76cc28 namespace: kserve-ci-e2e-test resourceVersion: "22449" uid: abe11d70-5002-4b84-b8f5-1adac8b593ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:10Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e90df namespace: kserve-ci-e2e-test resourceVersion: "22447" uid: d015d876-1fc8-4f65-9fc3-5d08253cf634 kind: Event lastTimestamp: "2026-09-08T13:41:10Z" message: InferenceService [error-404-isvc-e90df] is Ready metadata: creationTimestamp: "2026-09-08T13:41:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:10Z" name: error-404-isvc-e90df.18d35c379e082163 namespace: kserve-ci-e2e-test resourceVersion: "22448" uid: e12ca9d9-4482-4dd7-8d51-df61283c31fa reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:28:58.292523Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15946" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-f794f-predictor-f4899d6f4-94s6d to ip-10-0-141-177.ec2.internal metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:28:58Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b8d08cf3555 namespace: kserve-ci-e2e-test resourceVersion: "15952" uid: 3ecc5901-007b-466a-acd3-dd3470f71a84 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-f794f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:58Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b8d10dc4f13 namespace: kserve-ci-e2e-test resourceVersion: "15973" uid: c1a019ef-e9f0-448a-9235-8275511ffc42 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15957" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:28:59Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b8d4e6459f2 namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: e4274602-82a4-4208-bf79-047ab48f6182 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b8d4fbbd40b namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: bed63572-1a56-43c3-a632-670862718a47 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:13Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1946" in 13.681s (13.681s including waiting). Image size: 1355705921 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:13Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b907f3385be namespace: kserve-ci-e2e-test resourceVersion: "16169" uid: 2abd793f-6822-439f-9447-8868dfd99bdb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:13Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b9082ca763e namespace: kserve-ci-e2e-test resourceVersion: "16171" uid: 29a81297-d112-4c85-a731-6397ac8a97d2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:13Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b90832e7b8e namespace: kserve-ci-e2e-test resourceVersion: "16172" uid: 93145b81-c1f0-4988-b186-71750e43c6f5 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:13Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:13Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b90833b31d3 namespace: kserve-ci-e2e-test resourceVersion: "16173" uid: d2569645-0792-4322-a45b-d3bf4e0248cb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:15Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.2s (2.2s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:15Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b9106610c98 namespace: kserve-ci-e2e-test resourceVersion: "16196" uid: 9bedc53c-a871-4901-9074-d5409e0f8c25 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:15Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b9109abcd29 namespace: kserve-ci-e2e-test resourceVersion: "16197" uid: b5790389-f4ef-4061-9d4a-7fa19c75bb38 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:15Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b910a0dbdd6 namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 32e531cf-319b-4277-9ec2-510d2070adb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:29:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:29:51Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:29:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:51Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35b91166004e5 namespace: kserve-ci-e2e-test resourceVersion: "16529" uid: 95ce4c1a-082d-42d2-97b6-363364fb3d64 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35ba2e5ad17ec namespace: kserve-ci-e2e-test resourceVersion: "16995" uid: 726d1cbb-f1ee-408d-8dac-a82746eb42db reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d namespace: kserve-ci-e2e-test resourceVersion: "15948" uid: 15cac488-fad7-4af0-ba52-bd9c149ba2d8 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: error-404-isvc-f794f-predictor-f4899d6f4-94s6d.18d35ba2e5addc43 namespace: kserve-ci-e2e-test resourceVersion: "16997" uid: eee6ce56-47a1-4cc8-ac1e-7284a54342fc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-141-177.ec2.internal source: component: kubelet host: ip-10-0-141-177.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-f794f-predictor-f4899d6f4 namespace: kserve-ci-e2e-test resourceVersion: "15939" uid: e8063ea6-36e2-4515-92bb-50343ca6262e kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'Created pod: error-404-isvc-f794f-predictor-f4899d6f4-94s6d' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: error-404-isvc-f794f-predictor-f4899d6f4.18d35b8d083b5a60 namespace: kserve-ci-e2e-test resourceVersion: "15947" uid: d1a60f12-3bb1-4da5-a162-240d594ea1d2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15936" uid: 652b51e8-e97c-41f2-b808-2c0cfd47a70d kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: Scaled up replica set error-404-isvc-f794f-predictor-f4899d6f4 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: error-404-isvc-f794f-predictor.18d35b8d0615267c namespace: kserve-ci-e2e-test resourceVersion: "15941" uid: 595ca487-b6a4-4001-98c5-fc26f274b890 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: b2df8ca2-715f-48d4-89a4-6157045807c6 kind: Event lastTimestamp: "2026-09-08T13:29:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:43Z" name: error-404-isvc-f794f-predictor.18d35b90849661ff namespace: kserve-ci-e2e-test resourceVersion: "16465" uid: ff6d1956-9440-45e6-beb3-0ad98afa6d03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15938" uid: b2df8ca2-715f-48d4-89a4-6157045807c6 kind: Event lastTimestamp: "2026-09-08T13:29:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:43Z" name: error-404-isvc-f794f-predictor.18d35b9084d978ad namespace: kserve-ci-e2e-test resourceVersion: "16466" uid: 2d8f5759-775c-40cf-98bc-80d1a3f71cb3 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:29:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: b2df8ca2-715f-48d4-89a4-6157045807c6 kind: Event lastTimestamp: "2026-09-08T13:30:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:13Z" name: error-404-isvc-f794f-predictor.18d35b9b010894d7 namespace: kserve-ci-e2e-test resourceVersion: "16727" uid: b2aee575-2d06-41d3-bba3-6fc523133313 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:29:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16174" uid: b2df8ca2-715f-48d4-89a4-6157045807c6 kind: Event lastTimestamp: "2026-09-08T13:30:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:13Z" name: error-404-isvc-f794f-predictor.18d35b9b017fc4c4 namespace: kserve-ci-e2e-test resourceVersion: "16728" uid: 98d83596-2aa9-4f72-8720-357e2efdbf86 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f794f namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 9f10c4c7-54f0-45d0-acde-bd44a8c61640 kind: Event lastTimestamp: "2026-09-08T13:30:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:01Z" name: error-404-isvc-f794f.18d35b8d06224459 namespace: kserve-ci-e2e-test resourceVersion: "16623" uid: 02d3e7d9-9fb9-44fb-8e2e-57fa69f01375 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-f794f namespace: kserve-ci-e2e-test resourceVersion: "16621" uid: 9f10c4c7-54f0-45d0-acde-bd44a8c61640 kind: Event lastTimestamp: "2026-09-08T13:30:01Z" message: InferenceService [error-404-isvc-f794f] is Ready metadata: creationTimestamp: "2026-09-08T13:30:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:01Z" name: error-404-isvc-f794f.18d35b9bd207a25f namespace: kserve-ci-e2e-test resourceVersion: "16622" uid: e1d9bc09-391c-438b-a7a6-05f96fe4762f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:28:58.006778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15891" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8cf7c70e88 namespace: kserve-ci-e2e-test resourceVersion: "15895" uid: fac835dc-4055-41bd-ba27-445207feface reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8cfea4074b namespace: kserve-ci-e2e-test resourceVersion: "15916" uid: aafd6620-c79b-4d1d-97e6-219ffc4bd829 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15906" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8d3dca8177 namespace: kserve-ci-e2e-test resourceVersion: "16010" uid: 12a2f4fc-e23e-4222-8231-6e1d0b0a7f07 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5ea09478793377febcb6cdc9ccdf0f01773e1155c9dff7d28ca581c50565149f" already present on machine metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8d3f53858b namespace: kserve-ci-e2e-test resourceVersion: "16012" uid: 0c11b43a-8386-4ce4-b406-236512a7aa36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8d42f666be namespace: kserve-ci-e2e-test resourceVersion: "16013" uid: 859e8391-edb2-435c-a5d6-f4d3222ab08e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8d4360e034 namespace: kserve-ci-e2e-test resourceVersion: "16014" uid: d0f207bf-86ab-4c84-91b2-c163034cd8b6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1946" metadata: creationTimestamp: "2026-09-08T13:29:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:04Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b8e82b42aee namespace: kserve-ci-e2e-test resourceVersion: "16089" uid: 3b832c55-795e-4164-acfa-29ef7bea50d1 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1946" in 15.105s (15.105s including waiting). Image size: 1583067020 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b920713d80e namespace: kserve-ci-e2e-test resourceVersion: "16238" uid: 79bc8df0-a484-47b9-8d6c-a9c47353ff9a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b920de22cf1 namespace: kserve-ci-e2e-test resourceVersion: "16241" uid: a84a7277-b982-421d-83df-cc9b9ab3f921 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b920f6fb8ba namespace: kserve-ci-e2e-test resourceVersion: "16242" uid: 815a3066-d7d6-4a06-b580-fdebe35c6561 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b920f8989ed namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: c76334bb-f321-4878-ac15-29f4045429cc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b9216818cf6 namespace: kserve-ci-e2e-test resourceVersion: "16246" uid: 307ad2a9-301a-43ba-bad0-e9abebf55a72 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:29:20Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:20Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b9217cf0071 namespace: kserve-ci-e2e-test resourceVersion: "16247" uid: a12769da-0cd4-4e21-a4b2-7649eeb14276 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-08T13:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:30:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:26Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35b9241defccf namespace: kserve-ci-e2e-test resourceVersion: "16861" uid: ae99d9f7-dff9-45fb-8260-d3ebc8ab4005 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:31:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:07Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35bab3b886a41 namespace: kserve-ci-e2e-test resourceVersion: "17472" uid: e64f33df-888b-41e6-b1f9-2c3f0814fb23 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:31:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:07Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35bab3b88e1a5 namespace: kserve-ci-e2e-test resourceVersion: "17477" uid: d410831f-de18-4e1f-b105-1e99442d6d4f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9 namespace: kserve-ci-e2e-test resourceVersion: "15893" uid: 9deb1efd-bcdf-40e2-9410-0a86324765a1 kind: Event lastTimestamp: "2026-09-08T13:31:11Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:31:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:11Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9.18d35bac1a8cb433 namespace: kserve-ci-e2e-test resourceVersion: "17601" uid: 9c79b88a-6457-4f03-803c-cafa270aa658 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-7f476ffc7 namespace: kserve-ci-e2e-test resourceVersion: "15881" uid: 81b92892-055a-480b-a2c5-14aabefcc971 kind: Event lastTimestamp: "2026-09-08T13:28:57Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-7f476ffc7-bw8k9' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-1-predictor-7f476ffc7.18d35b8cf749ec8f namespace: kserve-ci-e2e-test resourceVersion: "15892" uid: 8814b5a3-efda-41cb-bf89-b12f776f7028 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15879" uid: d3f42e3d-e01e-41c1-9123-eed22e16945b kind: Event lastTimestamp: "2026-09-08T13:28:57Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-7f476ffc7 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:57Z" name: isvc-sklearn-graph-1-predictor.18d35b8cf3da4504 namespace: kserve-ci-e2e-test resourceVersion: "15882" uid: c3474914-2eea-4699-b9dc-9e66be67e14a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: ef5558d8-0601-4d42-8389-73424bf33d52 kind: Event lastTimestamp: "2026-09-08T13:29:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:42Z" name: isvc-sklearn-graph-1-predictor.18d35b90725989e5 namespace: kserve-ci-e2e-test resourceVersion: "16460" uid: d44ad506-acfc-4ae1-bfd8-0501cddd5019 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "15880" uid: ef5558d8-0601-4d42-8389-73424bf33d52 kind: Event lastTimestamp: "2026-09-08T13:29:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:42Z" name: isvc-sklearn-graph-1-predictor.18d35b9072cd81c1 namespace: kserve-ci-e2e-test resourceVersion: "16461" uid: 65162a96-a78f-40f5-8c16-ff3e026947a5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:29:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: ef5558d8-0601-4d42-8389-73424bf33d52 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-sklearn-graph-1-predictor.18d35b9aef4e5d1a namespace: kserve-ci-e2e-test resourceVersion: "17316" uid: 56b3f28b-a74f-40a6-b04e-5978f9223153 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:29:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "16162" uid: ef5558d8-0601-4d42-8389-73424bf33d52 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-sklearn-graph-1-predictor.18d35b9aefc687ab namespace: kserve-ci-e2e-test resourceVersion: "17317" uid: 033cb4e3-4251-4421-a7cc-563e90f0ddd8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15862" uid: fa888221-24bc-4eba-a7ce-0c1de22dd041 kind: Event lastTimestamp: "2026-09-08T13:30:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:36Z" name: isvc-sklearn-graph-1.18d35b8cf422a887 namespace: kserve-ci-e2e-test resourceVersion: "17115" uid: 96b41aad-8e9b-4b00-a701-cede29703740 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "17113" uid: fa888221-24bc-4eba-a7ce-0c1de22dd041 kind: Event lastTimestamp: "2026-09-08T13:30:36Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-08T13:30:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:36Z" name: isvc-sklearn-graph-1.18d35ba3fae6fe1c namespace: kserve-ci-e2e-test resourceVersion: "17114" uid: 60090733-5fb5-4bc9-9983-07485a957787 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:28:58.476804Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15976" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-766ff5986c-vkntk to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8d13cb1665 namespace: kserve-ci-e2e-test resourceVersion: "15980" uid: c44c6d03-7d6f-4db7-862e-b4a4f926786f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8d1e63c085 namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: d2781da7-f7c0-481d-8112-b9dde0ff31de reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Add eth0 [10.132.0.16/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8d596bd474 namespace: kserve-ci-e2e-test resourceVersion: "16031" uid: e83efff2-fee9-4884-9722-eb833697c8a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Pulling image "quay.io/opendatahub/kserve-storage-initializer@sha256:5ea09478793377febcb6cdc9ccdf0f01773e1155c9dff7d28ca581c50565149f" metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8d5acfc318 namespace: kserve-ci-e2e-test resourceVersion: "16035" uid: 654ba390-51c3-4618-96e6-b22e5ff7d503 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:02Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-storage-initializer@sha256:5ea09478793377febcb6cdc9ccdf0f01773e1155c9dff7d28ca581c50565149f" in 3.033s (3.034s including waiting). Image size: 308572071 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:02Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8e0fa760fc namespace: kserve-ci-e2e-test resourceVersion: "16067" uid: b3584c86-b045-4cec-ac7d-8adfb535698b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:02Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-08T13:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:02Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8e12ffb3fd namespace: kserve-ci-e2e-test resourceVersion: "16068" uid: 3587d10e-7121-4d46-b586-a2b26c647402 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:02Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:02Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-08T13:29:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:02Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8e13777527 namespace: kserve-ci-e2e-test resourceVersion: "16069" uid: 2bb50b51-38b8-4cd7-9580-add9cdb1c501 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:06Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1946" metadata: creationTimestamp: "2026-09-08T13:29:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:06Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b8f00ea1140 namespace: kserve-ci-e2e-test resourceVersion: "16111" uid: a2d57a60-9c12-4918-8e9a-5b4d9afb633a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1946" in 12.87s (12.87s including waiting). Image size: 1583067020 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b92000de864 namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: a9e9a0c1-5045-44b2-9f0c-a69a833e19b9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b9202eaec55 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 0963a9a2-2976-46bc-8692-8102e0510ce9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b92035c176b namespace: kserve-ci-e2e-test resourceVersion: "16235" uid: dd34cc92-9871-4917-bfd0-d7e03fa4e4fb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b920369b497 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 079cacdd-38d0-438a-8c21-b5b63f77f649 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:22Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.533s (2.533s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:22Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b929a666ac2 namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 98225992-1d1f-4f88-a893-cd4a1eb4102a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:22Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:22Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b929d32e069 namespace: kserve-ci-e2e-test resourceVersion: "16276" uid: 6ac6f89c-9fb4-499a-ac58-4e2be85884e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:29:22Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:29:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:22Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b929da30d2b namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 91aef27f-73b5-44cb-8efc-a7cc4973a13e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-09-08T13:29:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Readiness probe failed: dial tcp 10.132.0.16:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35b92fb0197f7 namespace: kserve-ci-e2e-test resourceVersion: "17576" uid: 8ca3b21a-533a-426e-a693-e2c4cd68e2be reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:31:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:07Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35bab394e3858 namespace: kserve-ci-e2e-test resourceVersion: "17454" uid: 45ec6bd7-6b79-42c5-a6b6-bc21e0b54484 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:31:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:07Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35bab394eef47 namespace: kserve-ci-e2e-test resourceVersion: "17459" uid: 60ea2412-a978-4173-8db9-1590fbfab13e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk namespace: kserve-ci-e2e-test resourceVersion: "15978" uid: 74d022bb-2968-40e9-932f-fe49a9c2885e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Readiness probe failed: Get "https://10.132.0.16:8643/healthz": dial tcp 10.132.0.16:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk.18d35baba9414852 namespace: kserve-ci-e2e-test resourceVersion: "17575" uid: cb159c08-5af1-4dea-91ea-9d64411aea5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-766ff5986c namespace: kserve-ci-e2e-test resourceVersion: "15967" uid: d8e740ad-5f7a-4a09-9a21-fed27786b4a3 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-766ff5986c-vkntk' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-2-predictor-766ff5986c.18d35b8d1348724b namespace: kserve-ci-e2e-test resourceVersion: "15977" uid: d8645385-6d98-4617-8c70-6382f15a4e9d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15965" uid: 0e4f82b7-de1b-43f3-abfc-cbd5965b553c kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-766ff5986c from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: isvc-sklearn-graph-2-predictor.18d35b8d0eab6df7 namespace: kserve-ci-e2e-test resourceVersion: "15968" uid: d35150e2-268c-499c-8d7b-9032a761775e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 42eba3bf-c5ed-4a53-af93-7f5389ac2512 kind: Event lastTimestamp: "2026-09-08T13:29:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:43Z" name: isvc-sklearn-graph-2-predictor.18d35b908d3ec81a namespace: kserve-ci-e2e-test resourceVersion: "16468" uid: d9b95134-59c1-4750-ae93-ac0d462499ca reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "15966" uid: 42eba3bf-c5ed-4a53-af93-7f5389ac2512 kind: Event lastTimestamp: "2026-09-08T13:29:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:43Z" name: isvc-sklearn-graph-2-predictor.18d35b908db55495 namespace: kserve-ci-e2e-test resourceVersion: "16469" uid: 74941318-98a9-495f-a5bf-9ddb7672106c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:29:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 42eba3bf-c5ed-4a53-af93-7f5389ac2512 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-sklearn-graph-2-predictor.18d35b9b0a17b24c namespace: kserve-ci-e2e-test resourceVersion: "17326" uid: 11a65b04-fc00-4da9-a3e0-61444cd34c02 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:29:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "16179" uid: 42eba3bf-c5ed-4a53-af93-7f5389ac2512 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-sklearn-graph-2-predictor.18d35b9b0a889c6f namespace: kserve-ci-e2e-test resourceVersion: "17327" uid: c22823ee-d163-4d14-9355-0df2748af6da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15951" uid: c37730a1-17a1-4f08-bee0-1e81f6e1f21c kind: Event lastTimestamp: "2026-09-08T13:30:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:39Z" name: isvc-sklearn-graph-2.18d35b8d0ed762c7 namespace: kserve-ci-e2e-test resourceVersion: "17143" uid: 6aac1897-4c92-40af-9e1b-900fdb981718 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "17141" uid: c37730a1-17a1-4f08-bee0-1e81f6e1f21c kind: Event lastTimestamp: "2026-09-08T13:30:39Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-08T13:30:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:39Z" name: isvc-sklearn-graph-2.18d35ba4b3b39d2d namespace: kserve-ci-e2e-test resourceVersion: "17142" uid: 5021bb8e-e215-44c6-b1b8-0b8ea4d36e1b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:28:58.192092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15918" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-d85h2 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:28:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d02d2b2e7 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: 548915ec-4d2d-472c-a26b-66eba692f5cd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d0aad5972 namespace: kserve-ci-e2e-test resourceVersion: "15958" uid: 859fd2bf-f42d-44bc-be2b-afc0407ae251 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:28:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d486d575d namespace: kserve-ci-e2e-test resourceVersion: "16015" uid: 9b65d338-136a-4c02-a428-ff2c59ddaf8f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:5ea09478793377febcb6cdc9ccdf0f01773e1155c9dff7d28ca581c50565149f" already present on machine metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d49f2c878 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 83950632-8522-40a0-9015-40661b83edea reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d4e2dd155 namespace: kserve-ci-e2e-test resourceVersion: "16018" uid: 293f251d-9fb8-4f37-aa7b-a055b457a6ab reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:28:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-08T13:28:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:59Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8d4e9fcc54 namespace: kserve-ci-e2e-test resourceVersion: "16020" uid: f3bba74c-8765-43e4-9f37-44599a2997cd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:04Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-08T13:29:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:04Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b8e829b53fc namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 6dcf6f42-fb19-4447-8f0e-27ac2c0a8e89 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.727s (23.727s including waiting). Image size: 1332293375 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b9408dc1834 namespace: kserve-ci-e2e-test resourceVersion: "16331" uid: 3f955116-3e53-4deb-a8a4-6967e9caafdd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b940d4e8106 namespace: kserve-ci-e2e-test resourceVersion: "16335" uid: 4c741ab4-55c5-4783-80af-b4391642d9d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b940dbf9ab0 namespace: kserve-ci-e2e-test resourceVersion: "16336" uid: f5acd559-bd7b-4d43-9bc7-43ec89918207 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b940dc9bf30 namespace: kserve-ci-e2e-test resourceVersion: "16337" uid: d73ff232-556a-48d8-8781-e51f8119acd9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b9411275981 namespace: kserve-ci-e2e-test resourceVersion: "16339" uid: b1012c24-3111-4b5a-b7e5-56c16bd3e70c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:29:28Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b9411ca940a namespace: kserve-ci-e2e-test resourceVersion: "16340" uid: 1c68adff-02ce-4f65-b2ba-cc948f1e2780 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-08T13:29:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:30:24Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:24Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35b9421905cd8 namespace: kserve-ci-e2e-test resourceVersion: "16849" uid: 1ad8d4e1-44eb-4753-8372-15e6916b7c36 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35bab3d9eb2b1 namespace: kserve-ci-e2e-test resourceVersion: "17488" uid: ca4eb135-458f-41c7-8217-004a0a151f15 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35bab3d9f2199 namespace: kserve-ci-e2e-test resourceVersion: "17489" uid: 889beeed-fc22-494e-a223-4c7d80c234ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-d85h2 namespace: kserve-ci-e2e-test resourceVersion: "15922" uid: 83493665-53a4-4bed-8f89-0f2d1940bc18 kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": dial tcp 10.133.0.26:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: isvc-xgboost-graph-predictor-555fc69d56-d85h2.18d35baba5d10b06 namespace: kserve-ci-e2e-test resourceVersion: "17574" uid: 607b8776-dcb1-4ac9-b4ea-90215eaf2693 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 3a9c074a-49e5-4852-8497-4266a4b558df kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-d85h2' metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d35b8d01ed98c4 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 7621de1a-6c19-469c-b5c7-ad0037ebed0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 70660aa2-a495-4845-9d69-5724e18429a8 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:58Z" name: isvc-xgboost-graph-predictor.18d35b8cfb35a03b namespace: kserve-ci-e2e-test resourceVersion: "15910" uid: 522e8216-5611-4826-98f0-e4d48cc5bf42 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 8c0769b1-826d-4ab7-99ab-e96d1e5207d2 kind: Event lastTimestamp: "2026-09-08T13:29:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:58Z" name: isvc-xgboost-graph-predictor.18d35b90793aeec4 namespace: kserve-ci-e2e-test resourceVersion: "16582" uid: 387bf471-0804-4475-b1ca-f849fda2821a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:29:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 8c0769b1-826d-4ab7-99ab-e96d1e5207d2 kind: Event lastTimestamp: "2026-09-08T13:29:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:58Z" name: isvc-xgboost-graph-predictor.18d35b90799ba519 namespace: kserve-ci-e2e-test resourceVersion: "16583" uid: 6f9b4d03-977b-4195-93d0-12bea16ce2c5 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:30:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 8c0769b1-826d-4ab7-99ab-e96d1e5207d2 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:30:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-xgboost-graph-predictor.18d35b9e747c9ccf namespace: kserve-ci-e2e-test resourceVersion: "17318" uid: 22bdeb64-fda6-4a72-b8bf-1afeabc1266e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:30:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "16164" uid: 8c0769b1-826d-4ab7-99ab-e96d1e5207d2 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:30:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:58Z" name: isvc-xgboost-graph-predictor.18d35b9e74ed2305 namespace: kserve-ci-e2e-test resourceVersion: "17319" uid: 32136326-af27-4274-8ef2-a5801adf0279 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15890" uid: db9eb9f0-6b72-465a-a2fb-3cb1951701fc kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:34Z" name: isvc-xgboost-graph.18d35b8cfb73e127 namespace: kserve-ci-e2e-test resourceVersion: "17079" uid: 9f8057c6-5750-4cb1-805e-34b34cc66fea reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "17075" uid: db9eb9f0-6b72-465a-a2fb-3cb1951701fc kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:34Z" name: isvc-xgboost-graph.18d35ba386d429e2 namespace: kserve-ci-e2e-test resourceVersion: "17077" uid: 3bd86a5b-2b29-4e1f-9a9f-861aa1f4d5b4 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:30:57.743639Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17297" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-546cb8556c-kwp6s to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:30:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:30:57Z" name: model-chainer-546cb8556c-kwp6s.18d35ba8d8a6908f namespace: kserve-ci-e2e-test resourceVersion: "17302" uid: 50fff22a-15c8-4557-aa27-ae121fa3a8dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:57Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:30:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:57Z" name: model-chainer-546cb8556c-kwp6s.18d35ba8df9b9256 namespace: kserve-ci-e2e-test resourceVersion: "17312" uid: 4cea12f5-8cb8-4e3e-ab95-4239cc64a45f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:58Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17305" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: Add eth0 [10.132.0.18/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:30:58Z" name: model-chainer-546cb8556c-kwp6s.18d35ba91e8d20ea namespace: kserve-ci-e2e-test resourceVersion: "17332" uid: 35466491-e04a-4fa5-82a0-7b5d5da619c1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:30:58Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:30:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:58Z" name: model-chainer-546cb8556c-kwp6s.18d35ba91fd644b4 namespace: kserve-ci-e2e-test resourceVersion: "17334" uid: 3c963c85-16ec-4807-81bf-9ce1f5885937 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:30:59Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-08T13:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:59Z" name: model-chainer-546cb8556c-kwp6s.18d35ba923acd4be namespace: kserve-ci-e2e-test resourceVersion: "17335" uid: e9893d28-1aaf-4016-8e88-004aa928bcec reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:30:59Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-08T13:30:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:59Z" name: model-chainer-546cb8556c-kwp6s.18d35ba924290daf namespace: kserve-ci-e2e-test resourceVersion: "17336" uid: 448b494f-7185-4e61-917a-eb094522fb1b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:31:07Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-08T13:31:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:07Z" name: model-chainer-546cb8556c-kwp6s.18d35bab32eb22ab namespace: kserve-ci-e2e-test resourceVersion: "17431" uid: d7ac22b5-6162-4e06-a10f-9b29151e982e reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-546cb8556c-kwp6s namespace: kserve-ci-e2e-test resourceVersion: "17300" uid: c281bc12-19c2-4af6-874d-0c23e4b1d9a1 kind: Event lastTimestamp: "2026-09-08T13:31:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:35Z" name: model-chainer-546cb8556c-kwp6s.18d35babbd6adb5a namespace: kserve-ci-e2e-test resourceVersion: "17856" uid: 2c2be18f-9c41-40f7-8f8a-ccc2e19aa47f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-546cb8556c namespace: kserve-ci-e2e-test resourceVersion: "17294" uid: a650a549-7ba5-49e0-839f-1f3d8a0d71bd kind: Event lastTimestamp: "2026-09-08T13:30:57Z" message: 'Created pod: model-chainer-546cb8556c-kwp6s' metadata: creationTimestamp: "2026-09-08T13:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:57Z" name: model-chainer-546cb8556c.18d35ba8d8497861 namespace: kserve-ci-e2e-test resourceVersion: "17299" uid: 9cc8ddf0-2243-409a-b8e5-207f130bc59a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17292" uid: 2f4825d2-61ee-43ad-a4bc-64ca6d87cf3a kind: Event lastTimestamp: "2026-09-08T13:30:57Z" message: Scaled up replica set model-chainer-546cb8556c from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:57Z" name: model-chainer.18d35ba8d766bfa8 namespace: kserve-ci-e2e-test resourceVersion: "17295" uid: ac6b4d43-90bf-4c60-b90c-abf29e695e44 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:57Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "17308" uid: 2a5b9612-db9d-4bcb-bfb4-b09fdb3d2d2f kind: Event lastTimestamp: "2026-09-08T13:30:57Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-08T13:30:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:57Z" name: model-chainer.18d35ba8dc8bb220 namespace: kserve-ci-e2e-test resourceVersion: "17310" uid: 0081f07c-d971-469d-934f-4924a57b7d31 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:32:08.039778Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18142" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-30f77-859fc5c779-w9th4 to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35bb936a08bad namespace: kserve-ci-e2e-test resourceVersion: "18147" uid: c4868f23-0c8b-4a01-a084-24278fd6c1e2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18151" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: Add eth0 [10.132.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35bb95877bca3 namespace: kserve-ci-e2e-test resourceVersion: "18166" uid: b7c4c3a1-9a0d-4826-81cf-5d41a183693c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30f77} kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35bb959bdb67c namespace: kserve-ci-e2e-test resourceVersion: "18168" uid: 0a47e0dc-f592-4943-b779-8555231b67b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30f77} kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: 'Created container: sequence-graph-30f77' metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35bb95cf9cab1 namespace: kserve-ci-e2e-test resourceVersion: "18169" uid: 5c171e9b-94b7-4440-95a6-1a241de29f76 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30f77} kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: Started container sequence-graph-30f77 metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35bb95d86855f namespace: kserve-ci-e2e-test resourceVersion: "18172" uid: 5fe75685-8a82-4bef-9936-c5a4cc38a861 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30f77} kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Stopping container sequence-graph-30f77 metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:22Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35c2c5f81007d namespace: kserve-ci-e2e-test resourceVersion: "21724" uid: 4204c94c-454b-4f30-a798-3665360958d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-30f77} kind: Pod name: sequence-graph-30f77-859fc5c779-w9th4 namespace: kserve-ci-e2e-test resourceVersion: "18144" uid: 81c6b279-d313-4eb4-ad16-c43cf5d4472d kind: Event lastTimestamp: "2026-09-08T13:40:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:50Z" name: sequence-graph-30f77-859fc5c779-w9th4.18d35c2d05b33aca namespace: kserve-ci-e2e-test resourceVersion: "22127" uid: f15e39b7-d468-4d66-bf38-d16b3ed7eb53 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-30f77-859fc5c779 namespace: kserve-ci-e2e-test resourceVersion: "18139" uid: 7e37ab0a-5d2b-4281-a524-274f35c4f947 kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: 'Created pod: sequence-graph-30f77-859fc5c779-w9th4' metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77-859fc5c779.18d35bb93637cac9 namespace: kserve-ci-e2e-test resourceVersion: "18143" uid: f86a086d-81af-4b3b-b80a-dd02222448cd reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18137" uid: 021c65a4-2889-4851-ada2-63f5f0ca4ac8 kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: Scaled up replica set sequence-graph-30f77-859fc5c779 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77.18d35bb9356a48bf namespace: kserve-ci-e2e-test resourceVersion: "18140" uid: baf44a6b-ecca-4c7e-b35c-c04d9e315d03 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:32:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18153" uid: 774bdc82-d2d5-475f-bb99-c285c6c0f016 kind: Event lastTimestamp: "2026-09-08T13:32:08Z" message: InferenceGraph [sequence-graph-30f77] is Ready metadata: creationTimestamp: "2026-09-08T13:32:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:32:08Z" name: sequence-graph-30f77.18d35bb93a5c1248 namespace: kserve-ci-e2e-test resourceVersion: "18155" uid: 747974f2-68e8-4726-ad19-2d63af6bad6c reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:32:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: 1b9dbcb8-c969-4a54-baef-739321dce9e1 kind: Event lastTimestamp: "2026-09-08T13:33:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:33:08Z" name: sequence-graph-30f77.18d35bbcb3dc9beb namespace: kserve-ci-e2e-test resourceVersion: "18600" uid: ed961645-2e7c-41eb-9544-23cf26b43a43 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:32:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18138" uid: 1b9dbcb8-c969-4a54-baef-739321dce9e1 kind: Event lastTimestamp: "2026-09-08T13:33:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:32:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:33:08Z" name: sequence-graph-30f77.18d35bbcb448a551 namespace: kserve-ci-e2e-test resourceVersion: "18601" uid: d6941820-a1bd-45e5-9c2d-3aa221e20b38 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:42:33.258818Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23377" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c7da0-7b7778f46-tqcxt to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:42:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:42:33Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4ac895f873 namespace: kserve-ci-e2e-test resourceVersion: "23382" uid: 4d2d46ef-8d70-44d4-9b8e-9484a8aa42a9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:33Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:42:33Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-c7da0-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:33Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4ad2a872c5 namespace: kserve-ci-e2e-test resourceVersion: "23395" uid: 1b25327a-60ba-4463-9126-96ad9c36f54a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:34Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23386" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:42:34Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:42:34Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4b0decbdc5 namespace: kserve-ci-e2e-test resourceVersion: "23404" uid: 424e877e-921a-4d5f-a9f2-1ae27130e0f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7da0} kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:42:34Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:34Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4b0fee9943 namespace: kserve-ci-e2e-test resourceVersion: "23407" uid: e1c1e791-6a9b-4a2a-8187-291e310c359b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7da0} kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:42:34Z" message: 'Created container: sequence-graph-c7da0' metadata: creationTimestamp: "2026-09-08T13:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:34Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4b12b4462d namespace: kserve-ci-e2e-test resourceVersion: "23408" uid: f906f5e5-463c-4b81-a5cf-13f069d9aace reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7da0} kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:42:34Z" message: Started container sequence-graph-c7da0 metadata: creationTimestamp: "2026-09-08T13:42:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:34Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35c4b13336b3d namespace: kserve-ci-e2e-test resourceVersion: "23409" uid: d8e78841-9b3b-4d22-8085-b56d68bc6273 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7da0} kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Stopping container sequence-graph-c7da0 metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:47Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35cbde817e848 namespace: kserve-ci-e2e-test resourceVersion: "27265" uid: 741b804a-6a7b-4b1e-b417-2908e0a2fbc4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c7da0} kind: Pod name: sequence-graph-c7da0-7b7778f46-tqcxt namespace: kserve-ci-e2e-test resourceVersion: "23379" uid: 0d540e90-6dc4-4bc4-b048-e6205e12d0fc kind: Event lastTimestamp: "2026-09-08T13:51:15Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:15Z" name: sequence-graph-c7da0-7b7778f46-tqcxt.18d35cbe94f0e123 namespace: kserve-ci-e2e-test resourceVersion: "27668" uid: f7427099-45d6-4de9-9b7d-9d2713b34a47 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c7da0-7b7778f46 namespace: kserve-ci-e2e-test resourceVersion: "23374" uid: 84521f0e-ef87-464c-b72f-88ed847dc4f9 kind: Event lastTimestamp: "2026-09-08T13:42:33Z" message: 'Created pod: sequence-graph-c7da0-7b7778f46-tqcxt' metadata: creationTimestamp: "2026-09-08T13:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:33Z" name: sequence-graph-c7da0-7b7778f46.18d35c4ac7f5ff26 namespace: kserve-ci-e2e-test resourceVersion: "23380" uid: 6b52ccaf-da1c-4b35-865b-6a7931188f8e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23372" uid: 1b24a7cc-875b-45e3-a72f-e208b9434934 kind: Event lastTimestamp: "2026-09-08T13:42:33Z" message: Scaled up replica set sequence-graph-c7da0-7b7778f46 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:33Z" name: sequence-graph-c7da0.18d35c4ac6d4b8a1 namespace: kserve-ci-e2e-test resourceVersion: "23375" uid: 4b341c78-907f-4a22-9a79-813119750a1d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:33Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23388" uid: 00085be4-8e1d-419a-9cbf-a769d3c9283e kind: Event lastTimestamp: "2026-09-08T13:42:33Z" message: InferenceGraph [sequence-graph-c7da0] is Ready metadata: creationTimestamp: "2026-09-08T13:42:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:42:33Z" name: sequence-graph-c7da0.18d35c4acd402b68 namespace: kserve-ci-e2e-test resourceVersion: "23390" uid: 1a091d2d-b5ee-408e-8ca1-a703c08e81b5 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:42:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23373" uid: 2bf52595-05ba-483f-abb5-1a162e731820 kind: Event lastTimestamp: "2026-09-08T13:43:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:43:03Z" name: sequence-graph-c7da0.18d35c4e458eb972 namespace: kserve-ci-e2e-test resourceVersion: "23622" uid: eda174e2-2eaf-43dd-adcb-5c93ee477e83 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:42:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23373" uid: 2bf52595-05ba-483f-abb5-1a162e731820 kind: Event lastTimestamp: "2026-09-08T13:43:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:42:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:43:03Z" name: sequence-graph-c7da0.18d35c4e461139dc namespace: kserve-ci-e2e-test resourceVersion: "23623" uid: 7ef2458f-3dfb-401a-b823-13e918eead68 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:41:22.897863Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22561" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-e90df-764845479d-h54j2 to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:41:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:41:22Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3a66bf0611 namespace: kserve-ci-e2e-test resourceVersion: "22567" uid: dc15fe67-0ec8-4956-a59b-b222b9e3aa16 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-e90df-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:23Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3a7433d8d9 namespace: kserve-ci-e2e-test resourceVersion: "22585" uid: 7cb9c87c-86d1-4b16-a0e1-e989ae453d30 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:24Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22570" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:24Z" message: Add eth0 [10.132.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:41:24Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3aac2eed4c namespace: kserve-ci-e2e-test resourceVersion: "22597" uid: 43c42995-43b5-4cbf-bcc4-53295fc5712e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e90df} kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:24Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:24Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3aadb13cd4 namespace: kserve-ci-e2e-test resourceVersion: "22599" uid: 945b6ed8-6e29-4d09-bf10-6e74f8a95d32 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e90df} kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:24Z" message: 'Created container: sequence-graph-e90df' metadata: creationTimestamp: "2026-09-08T13:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:24Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3ab070dc0d namespace: kserve-ci-e2e-test resourceVersion: "22600" uid: a4378db0-5c25-4311-b1b4-997140a8a8f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e90df} kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:24Z" message: Started container sequence-graph-e90df metadata: creationTimestamp: "2026-09-08T13:41:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:24Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3ab0e599b0 namespace: kserve-ci-e2e-test resourceVersion: "22601" uid: f95fab3c-b0c9-4c8a-a2c1-f0cad24682dd reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e90df} kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:41:32Z" message: Stopping container sequence-graph-e90df metadata: creationTimestamp: "2026-09-08T13:41:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:32Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3cc09b1c3b namespace: kserve-ci-e2e-test resourceVersion: "22694" uid: 6b96457d-a518-4d4a-a152-6a2125742b00 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:41:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-e90df} kind: Pod name: sequence-graph-e90df-764845479d-h54j2 namespace: kserve-ci-e2e-test resourceVersion: "22564" uid: b3edddb8-2e57-4e63-bb22-9736954c9204 kind: Event lastTimestamp: "2026-09-08T13:42:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:41:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:42:00Z" name: sequence-graph-e90df-764845479d-h54j2.18d35c3d4d8ceb8f namespace: kserve-ci-e2e-test resourceVersion: "23093" uid: 98450c12-6da3-4eb2-8e17-fba0b4b7f936 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-e90df-764845479d namespace: kserve-ci-e2e-test resourceVersion: "22558" uid: 2e35c4a7-df62-4690-82c4-483cc543950d kind: Event lastTimestamp: "2026-09-08T13:41:22Z" message: 'Created pod: sequence-graph-e90df-764845479d-h54j2' metadata: creationTimestamp: "2026-09-08T13:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:22Z" name: sequence-graph-e90df-764845479d.18d35c3a65e88dd9 namespace: kserve-ci-e2e-test resourceVersion: "22562" uid: 1d08f92c-c5bd-4b40-868f-527c7cdea4f1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-e90df namespace: kserve-ci-e2e-test resourceVersion: "22556" uid: ca7f4859-2441-48d2-8eb9-34e3dfe8006d kind: Event lastTimestamp: "2026-09-08T13:41:22Z" message: Scaled up replica set sequence-graph-e90df-764845479d from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:22Z" name: sequence-graph-e90df.18d35c3a64f4b164 namespace: kserve-ci-e2e-test resourceVersion: "22559" uid: 4ca3010c-c38c-463c-a3f7-f66fc1f0af97 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e90df namespace: kserve-ci-e2e-test resourceVersion: "22573" uid: e137bf5a-1fab-4d35-91cd-8023108ad5d8 kind: Event lastTimestamp: "2026-09-08T13:41:22Z" message: InferenceGraph [sequence-graph-e90df] is Ready metadata: creationTimestamp: "2026-09-08T13:41:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:22Z" name: sequence-graph-e90df.18d35c3a6c0cacc7 namespace: kserve-ci-e2e-test resourceVersion: "22575" uid: d53358bf-af07-45d5-af23-443197f8b854 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e90df namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: e137bf5a-1fab-4d35-91cd-8023108ad5d8 kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'Failed to update status for InferenceGraph "sequence-graph-e90df": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-e90df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:23Z" name: sequence-graph-e90df.18d35c3a6f00c865 namespace: kserve-ci-e2e-test resourceVersion: "22577" uid: 6e52d5fe-4385-4f15-ba1e-7a7f456257ce reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-e90df namespace: kserve-ci-e2e-test resourceVersion: "22551" uid: e137bf5a-1fab-4d35-91cd-8023108ad5d8 kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-e90df": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:23Z" name: sequence-graph-e90df.18d35c3a6f010dc9 namespace: kserve-ci-e2e-test resourceVersion: "22580" uid: 1108e4f8-a787-4006-ab18-94d768c57437 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:51:12.458373Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27622" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-13b40-64fb84556f-s67d4 to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:51:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:51:12Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3ab4a711b namespace: kserve-ci-e2e-test resourceVersion: "27627" uid: bd29e819-3538-4c68-979d-fe18bfc4e3e4 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:12Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-13b40-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:12Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3b3954521 namespace: kserve-ci-e2e-test resourceVersion: "27639" uid: b992bf50-f7dc-4873-81c7-c1b445d6ca5b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:13Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27631" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:13Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:51:13Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3f192468d namespace: kserve-ci-e2e-test resourceVersion: "27650" uid: 1dba5bf4-4064-4039-b69c-e7e45a9f63db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13b40} kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:13Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:13Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3f2f56baf namespace: kserve-ci-e2e-test resourceVersion: "27652" uid: 4137a621-f76b-4913-95af-c30226a1574e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13b40} kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:13Z" message: 'Created container: splitter-graph-13b40' metadata: creationTimestamp: "2026-09-08T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:13Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3f6a73c78 namespace: kserve-ci-e2e-test resourceVersion: "27653" uid: 7e96992a-a4ac-461b-a951-4884ea57a9b2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13b40} kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:13Z" message: Started container splitter-graph-13b40 metadata: creationTimestamp: "2026-09-08T13:51:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:13Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc3f720f6f6 namespace: kserve-ci-e2e-test resourceVersion: "27654" uid: 7b6a92d6-07ed-4d7b-90a6-482d4de7d3df reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13b40} kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Stopping container splitter-graph-13b40 metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:22Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc6030d5e77 namespace: kserve-ci-e2e-test resourceVersion: "27751" uid: 4cf9e71b-f80a-4e1c-ac1f-aa4518832ee3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-13b40} kind: Pod name: splitter-graph-13b40-64fb84556f-s67d4 namespace: kserve-ci-e2e-test resourceVersion: "27624" uid: 0553c5d9-6a94-4409-a358-32bac469b315 kind: Event lastTimestamp: "2026-09-08T13:51:50Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:50Z" name: splitter-graph-13b40-64fb84556f-s67d4.18d35cc6af16aa25 namespace: kserve-ci-e2e-test resourceVersion: "28152" uid: 31833fc6-c939-4022-85f1-eebb4613b0a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-13b40-64fb84556f namespace: kserve-ci-e2e-test resourceVersion: "27619" uid: 5f8ec611-1472-4279-afd3-d95fa03ed244 kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'Created pod: splitter-graph-13b40-64fb84556f-s67d4' metadata: creationTimestamp: "2026-09-08T13:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: splitter-graph-13b40-64fb84556f.18d35cc3aaab64ca namespace: kserve-ci-e2e-test resourceVersion: "27623" uid: bf2f4739-0c2c-405b-89d3-93faee94fc66 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-13b40 namespace: kserve-ci-e2e-test resourceVersion: "27617" uid: 348e44ad-8f29-4b3b-b831-1d336dcc7368 kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: Scaled up replica set splitter-graph-13b40-64fb84556f from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: splitter-graph-13b40.18d35cc3a9d5ac41 namespace: kserve-ci-e2e-test resourceVersion: "27620" uid: 327835e7-617f-40bf-bc09-5140a7f9faed reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:12Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-13b40 namespace: kserve-ci-e2e-test resourceVersion: "27633" uid: b2aa0d7c-9a5d-47e5-8a89-c574eb6441ca kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: InferenceGraph [splitter-graph-13b40] is Ready metadata: creationTimestamp: "2026-09-08T13:51:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:12Z" name: splitter-graph-13b40.18d35cc3afe19a58 namespace: kserve-ci-e2e-test resourceVersion: "27636" uid: 031f8b0d-ff07-4213-97c7-862aabc675aa reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:52:22.790782Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28435" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7d014-7f8d646646-jnp9w to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:52:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:52:22Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35cd40b6dd68b namespace: kserve-ci-e2e-test resourceVersion: "28440" uid: cb53c8c4-d25e-49ce-943c-3c66effb2795 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:23Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28444" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T13:52:23Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:52:23Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35cd42e155ea6 namespace: kserve-ci-e2e-test resourceVersion: "28459" uid: 18880667-c10d-4638-b2fa-e2d49e1cd0bd reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d014} kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28436" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T13:52:23Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:52:23Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35cd42f749cb7 namespace: kserve-ci-e2e-test resourceVersion: "28461" uid: 5b874ed7-c7e4-4d64-bba6-87abd628359e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d014} kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28436" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T13:52:23Z" message: 'Created container: splitter-graph-7d014' metadata: creationTimestamp: "2026-09-08T13:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:52:23Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35cd432903787 namespace: kserve-ci-e2e-test resourceVersion: "28463" uid: ef7e0f8f-1fb5-42b3-b6d8-c70cfc133870 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d014} kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28436" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T13:52:23Z" message: Started container splitter-graph-7d014 metadata: creationTimestamp: "2026-09-08T13:52:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:52:23Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35cd43308664e namespace: kserve-ci-e2e-test resourceVersion: "28464" uid: fc4a6303-8081-4c53-8658-10b985d4be15 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d014} kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28436" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T14:00:37Z" message: Stopping container splitter-graph-7d014 metadata: creationTimestamp: "2026-09-08T14:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:00:37Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35d47375c3923 namespace: kserve-ci-e2e-test resourceVersion: "31892" uid: 7714c0e5-9364-4cc2-b2f8-b40c0435b980 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T14:00:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7d014} kind: Pod name: splitter-graph-7d014-7f8d646646-jnp9w namespace: kserve-ci-e2e-test resourceVersion: "28436" uid: d28e1afb-10ba-4c97-9bfc-83c5bd6f8868 kind: Event lastTimestamp: "2026-09-08T14:01:04Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T14:00:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:01:04Z" name: splitter-graph-7d014-7f8d646646-jnp9w.18d35d47ba293e4d namespace: kserve-ci-e2e-test resourceVersion: "32120" uid: b0180626-067b-4e3a-9458-15e574025c4d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7d014-7f8d646646 namespace: kserve-ci-e2e-test resourceVersion: "28432" uid: 7f1d656e-b2ce-47c2-b9f6-25f9ff9c1ac8 kind: Event lastTimestamp: "2026-09-08T13:52:22Z" message: 'Created pod: splitter-graph-7d014-7f8d646646-jnp9w' metadata: creationTimestamp: "2026-09-08T13:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:22Z" name: splitter-graph-7d014-7f8d646646.18d35cd40b05bc33 namespace: kserve-ci-e2e-test resourceVersion: "28437" uid: 4f3ee2ae-df5f-402f-929f-70e1f8eea3e9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28430" uid: ae44f892-bd43-47a4-9d59-6749eed6d3e2 kind: Event lastTimestamp: "2026-09-08T13:52:22Z" message: Scaled up replica set splitter-graph-7d014-7f8d646646 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:22Z" name: splitter-graph-7d014.18d35cd40a21ff43 namespace: kserve-ci-e2e-test resourceVersion: "28433" uid: dacdf6e2-3af6-4fba-a095-cd6119e6ec2c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:22Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28448" uid: 6cea9973-46fe-4ab6-b310-9377dc8a45d1 kind: Event lastTimestamp: "2026-09-08T13:52:22Z" message: InferenceGraph [splitter-graph-7d014] is Ready metadata: creationTimestamp: "2026-09-08T13:52:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:52:22Z" name: splitter-graph-7d014.18d35cd410b06dcb namespace: kserve-ci-e2e-test resourceVersion: "28450" uid: 002a9a4b-8ac8-4440-b6e3-be3ba1634ee3 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28431" uid: a2463b7e-3655-4064-a6ab-af0049591bab kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: splitter-graph-7d014.18d35cd788c06ac1 namespace: kserve-ci-e2e-test resourceVersion: "28568" uid: 76d75f9c-6890-47b0-aa20-4f6922f54613 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28431" uid: a2463b7e-3655-4064-a6ab-af0049591bab kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: splitter-graph-7d014.18d35cd78915250e namespace: kserve-ci-e2e-test resourceVersion: "28570" uid: 47041391-ec26-4030-977c-45c931ad7946 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28567" uid: a2463b7e-3655-4064-a6ab-af0049591bab kind: Event lastTimestamp: "2026-09-08T13:52:52Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:52Z" name: splitter-graph-7d014.18d35cdb07d6e536 namespace: kserve-ci-e2e-test resourceVersion: "28682" uid: 8acb0882-2269-4d32-ba3b-419e6142d59c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:52Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28567" uid: a2463b7e-3655-4064-a6ab-af0049591bab kind: Event lastTimestamp: "2026-09-08T13:52:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:52Z" name: splitter-graph-7d014.18d35cdb0850bc7e namespace: kserve-ci-e2e-test resourceVersion: "28684" uid: 1ec331f8-68ac-4140-a800-f29683e868fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:50:12.414907Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-13b40-predictor-8bb687fb7-g48nb to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5b06bdae6 namespace: kserve-ci-e2e-test resourceVersion: "26897" uid: 88d3b318-8f41-452f-bac3-03cd6ebfddd2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26912" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5d2a5e43a namespace: kserve-ci-e2e-test resourceVersion: "26946" uid: abf0ba4d-058b-48e9-8aba-dae211ed6d12 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5d439cc98 namespace: kserve-ci-e2e-test resourceVersion: "26949" uid: 014d090d-abd9-4f41-9c3c-94df0d21c545 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5d78ca219 namespace: kserve-ci-e2e-test resourceVersion: "26950" uid: 37004e25-efa2-4dc7-b743-e7b39d189116 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5d80190a9 namespace: kserve-ci-e2e-test resourceVersion: "26951" uid: 90635e42-9b64-4014-82af-61167d6f8b55 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5d80c44f9 namespace: kserve-ci-e2e-test resourceVersion: "26952" uid: 419a3428-b7fe-47ce-90a8-80d8f6330c94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5db18b857 namespace: kserve-ci-e2e-test resourceVersion: "26958" uid: 9083d04c-39d6-456a-8b66-1fc3d90439f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:13Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:13Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb5db8f3a48 namespace: kserve-ci-e2e-test resourceVersion: "26959" uid: eac18024-3a67-424c-a0e2-a757bc96ab0c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cb60a477010 namespace: kserve-ci-e2e-test resourceVersion: "27401" uid: d98d5049-7e99-4951-aa8d-d0b353ed855d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:22Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cc609c6d9c9 namespace: kserve-ci-e2e-test resourceVersion: "27771" uid: cb38e780-4ae6-4fad-9c61-4b1fdf3a8308 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:22Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cc609c76024 namespace: kserve-ci-e2e-test resourceVersion: "27776" uid: e1a39dda-ff8c-4224-ab58-a202e6461e28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb namespace: kserve-ci-e2e-test resourceVersion: "26888" uid: 1655960d-4ef9-47b9-97de-c266d9e1d18e kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-13b40-predictor-8bb687fb7-g48nb.18d35cc6926cddfe namespace: kserve-ci-e2e-test resourceVersion: "27884" uid: c638c8a7-b543-49f6-9fe4-76efe3691875 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-13b40-predictor-8bb687fb7 namespace: kserve-ci-e2e-test resourceVersion: "26870" uid: 6fb29ff8-f3e2-4a63-b4d5-64e1efefddd8 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'Created pod: success-200-isvc-13b40-predictor-8bb687fb7-g48nb' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40-predictor-8bb687fb7.18d35cb5afb21a03 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: d2e05d9e-9e38-462b-8301-13f988c0a5e7 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26860" uid: e475b8db-77de-4fff-9516-ba36a6413703 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Scaled up replica set success-200-isvc-13b40-predictor-8bb687fb7 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40-predictor.18d35cb5aa18d2b3 namespace: kserve-ci-e2e-test resourceVersion: "26872" uid: 92b6ce33-7633-459c-b775-7d1efbf75adb reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: 1f2c8f53-e766-4d29-bb4e-7a387dae906f kind: Event lastTimestamp: "2026-09-08T13:50:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:42Z" name: success-200-isvc-13b40-predictor.18d35cb928a9b18a namespace: kserve-ci-e2e-test resourceVersion: "27208" uid: c84fd18c-cccd-4e8b-8b85-7887bf9d4aa1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:27Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "26869" uid: 1f2c8f53-e766-4d29-bb4e-7a387dae906f kind: Event lastTimestamp: "2026-09-08T13:50:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:50:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:42Z" name: success-200-isvc-13b40-predictor.18d35cb9291d9fe2 namespace: kserve-ci-e2e-test resourceVersion: "27209" uid: 450c5bef-63d2-4fb4-ab00-5fc695f8f7f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: 1f2c8f53-e766-4d29-bb4e-7a387dae906f kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: success-200-isvc-13b40-predictor.18d35cc02746e836 namespace: kserve-ci-e2e-test resourceVersion: "27611" uid: 0579d988-3e23-4e81-a060-e32b27552a4b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:50:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-13b40-predictor namespace: kserve-ci-e2e-test resourceVersion: "27088" uid: 1f2c8f53-e766-4d29-bb4e-7a387dae906f kind: Event lastTimestamp: "2026-09-08T13:51:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:50:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:12Z" name: success-200-isvc-13b40-predictor.18d35cc028054b9b namespace: kserve-ci-e2e-test resourceVersion: "27613" uid: 9e3cd452-0f2d-4a15-8e21-b92848b84271 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: 42a235f7-c8e7-457f-a1c5-2493a2bb3b42 kind: Event lastTimestamp: "2026-09-08T13:51:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:00Z" name: success-200-isvc-13b40.18d35cb5aa26625d namespace: kserve-ci-e2e-test resourceVersion: "27508" uid: 1ae8a678-95ac-4b22-835d-e6e4d959ba8f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: 42a235f7-c8e7-457f-a1c5-2493a2bb3b42 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'Failed to update status for InferenceService "success-200-isvc-13b40": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-13b40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40.18d35cb5aae42564 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: b4c5e116-7b54-4c55-aded-dab624b63459 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "26849" uid: 42a235f7-c8e7-457f-a1c5-2493a2bb3b42 kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-13b40": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-13b40.18d35cb5aae48dc1 namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 6207ba35-3259-472e-8640-995e10850921 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-13b40 namespace: kserve-ci-e2e-test resourceVersion: "27506" uid: 42a235f7-c8e7-457f-a1c5-2493a2bb3b42 kind: Event lastTimestamp: "2026-09-08T13:51:00Z" message: InferenceService [success-200-isvc-13b40] is Ready metadata: creationTimestamp: "2026-09-08T13:51:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:00Z" name: success-200-isvc-13b40.18d35cc0c740d02f namespace: kserve-ci-e2e-test resourceVersion: "27507" uid: cb3359af-c162-4d85-b055-b19fe4e8a47a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:31:08.113038Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17504" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-30f77-predictor-6b79cc6889-k52x5 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:31:08Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab42b70764 namespace: kserve-ci-e2e-test resourceVersion: "17508" uid: 4733bb05-d883-400e-ac43-c0c2d14fdf78 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-30f77-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:08Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab523e1927 namespace: kserve-ci-e2e-test resourceVersion: "17542" uid: e84dfddd-c99b-4d5a-94cf-6194d3a203cf reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17513" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab88a6cc06 namespace: kserve-ci-e2e-test resourceVersion: "17557" uid: 026aa1df-26b3-48ac-af11-154ae3abd6a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab8a37b583 namespace: kserve-ci-e2e-test resourceVersion: "17559" uid: 0746b228-f2c2-4561-b4f6-b00b71c6b82a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab8d098179 namespace: kserve-ci-e2e-test resourceVersion: "17560" uid: 3bcaec0f-a18b-4bc1-a3e5-38e17d2b1269 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab8d75185a namespace: kserve-ci-e2e-test resourceVersion: "17561" uid: e00ae717-1f3d-404e-93e9-9c6455dc8e14 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab8d81ca78 namespace: kserve-ci-e2e-test resourceVersion: "17562" uid: 8f1d3232-67a7-4483-8e0e-1d15cf904293 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab9051c5dd namespace: kserve-ci-e2e-test resourceVersion: "17566" uid: 9d093031-04b6-494b-929d-5cf59c415383 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:31:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:09Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35bab90b5a77d namespace: kserve-ci-e2e-test resourceVersion: "17567" uid: a0abfb77-b8f7-41b6-b4ef-1ea9ab44c2b3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:31:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:31:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:31:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:46Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35babc016b4cf namespace: kserve-ci-e2e-test resourceVersion: "17943" uid: 2008979a-8510-4162-a0b2-4a41cb172e28 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:22Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35c2c69c97a05 namespace: kserve-ci-e2e-test resourceVersion: "21747" uid: 867b8ad5-3f19-42b3-b1b8-f54489dae414 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:22Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35c2c69c9e2b3 namespace: kserve-ci-e2e-test resourceVersion: "21754" uid: c7a15b48-f32a-47de-8646-20b229c9a2ea reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:40:27Z" message: 'Readiness probe failed: Get "https://10.133.0.28:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-08T13:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:27Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35c2d701ff921 namespace: kserve-ci-e2e-test resourceVersion: "21888" uid: a28ddf0b-eba0-4eb4-a340-925b1a4ed7fd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5 namespace: kserve-ci-e2e-test resourceVersion: "17506" uid: 2a2e4e77-5919-4168-84f7-3d085eabbd6e kind: Event lastTimestamp: "2026-09-08T13:40:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: i/o timeout' metadata: creationTimestamp: "2026-09-08T13:40:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:27Z" name: success-200-isvc-30f77-predictor-6b79cc6889-k52x5.18d35c2d706e4497 namespace: kserve-ci-e2e-test resourceVersion: "21889" uid: d9ba1986-9a68-4948-b9d8-9e1a36eebf40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-30f77-predictor-6b79cc6889 namespace: kserve-ci-e2e-test resourceVersion: "17498" uid: 8248e720-8b65-42ae-a33c-eeb6b8257ae7 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: 'Created pod: success-200-isvc-30f77-predictor-6b79cc6889-k52x5' metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:08Z" name: success-200-isvc-30f77-predictor-6b79cc6889.18d35bab4241233d namespace: kserve-ci-e2e-test resourceVersion: "17505" uid: cae8a5d0-07ca-497e-98a2-eea9e430551e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17496" uid: f0ec52d9-b847-4d64-859d-0f42bd29e5d5 kind: Event lastTimestamp: "2026-09-08T13:31:08Z" message: Scaled up replica set success-200-isvc-30f77-predictor-6b79cc6889 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:08Z" name: success-200-isvc-30f77-predictor.18d35bab40925773 namespace: kserve-ci-e2e-test resourceVersion: "17499" uid: 9064e8ab-aaaf-429c-9c62-ee8c7bd34c49 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: 93a32d50-8a7b-4354-99ca-f1a6c1948e46 kind: Event lastTimestamp: "2026-09-08T13:31:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:23Z" name: success-200-isvc-30f77-predictor.18d35baebf0b4be6 namespace: kserve-ci-e2e-test resourceVersion: "17721" uid: a9faef7a-be13-4e2f-ae71-13e452f2e738 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17497" uid: 93a32d50-8a7b-4354-99ca-f1a6c1948e46 kind: Event lastTimestamp: "2026-09-08T13:31:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:23Z" name: success-200-isvc-30f77-predictor.18d35baebf68921b namespace: kserve-ci-e2e-test resourceVersion: "17723" uid: 4b15ea56-02c6-48c8-be3d-c4279411669f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: 93a32d50-8a7b-4354-99ca-f1a6c1948e46 kind: Event lastTimestamp: "2026-09-08T13:32:38Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:38Z" name: success-200-isvc-30f77-predictor.18d35bb23e812606 namespace: kserve-ci-e2e-test resourceVersion: "18387" uid: 2e831aad-550d-4689-9d4a-12615ca25a06 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:31:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-30f77-predictor namespace: kserve-ci-e2e-test resourceVersion: "17720" uid: 93a32d50-8a7b-4354-99ca-f1a6c1948e46 kind: Event lastTimestamp: "2026-09-08T13:32:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:38Z" name: success-200-isvc-30f77-predictor.18d35bb23eef29ee namespace: kserve-ci-e2e-test resourceVersion: "18388" uid: bed2980f-f12f-4e00-a560-e7975a04d43b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:31:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30f77 namespace: kserve-ci-e2e-test resourceVersion: "17487" uid: f9686c6c-2a6e-4c58-8341-b75bf953c65b kind: Event lastTimestamp: "2026-09-08T13:31:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:31:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:56Z" name: success-200-isvc-30f77.18d35bab40a3b295 namespace: kserve-ci-e2e-test resourceVersion: "18041" uid: 5a2312ca-27bc-4b51-b59f-35aa1f96db44 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-30f77 namespace: kserve-ci-e2e-test resourceVersion: "18039" uid: f9686c6c-2a6e-4c58-8341-b75bf953c65b kind: Event lastTimestamp: "2026-09-08T13:31:56Z" message: InferenceService [success-200-isvc-30f77] is Ready metadata: creationTimestamp: "2026-09-08T13:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:56Z" name: success-200-isvc-30f77.18d35bb67b30dfa2 namespace: kserve-ci-e2e-test resourceVersion: "18040" uid: b604f5dc-cdec-47f3-b8fe-793ead0f6c96 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:39:47.222482Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21346" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-601fd-predictor-646cfc64df-ckjl2 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c24200c9389 namespace: kserve-ci-e2e-test resourceVersion: "21356" uid: 4cf02831-6ceb-49bc-bc73-91d18b45139c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-601fd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c242c03478e namespace: kserve-ci-e2e-test resourceVersion: "21402" uid: 4d4e2777-3e3c-47b7-a199-3696bd885e93 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21363" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c24666883bd namespace: kserve-ci-e2e-test resourceVersion: "21426" uid: 9ae3f23f-26e9-4f51-870b-1f11f161e0e0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c2467fb5710 namespace: kserve-ci-e2e-test resourceVersion: "21428" uid: 526fcfa4-0b7f-4dd5-9fc1-008ae393cc96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c246b3b6a97 namespace: kserve-ci-e2e-test resourceVersion: "21429" uid: 7d11115c-15b5-4344-9c90-beeb85add293 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c246bab1fac namespace: kserve-ci-e2e-test resourceVersion: "21430" uid: 85d68577-e188-4610-ac4e-3351312d1c20 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c246bb50123 namespace: kserve-ci-e2e-test resourceVersion: "21431" uid: 4e8bdf7a-b642-4807-8308-6acc542c3ecc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c246ead0344 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 0314b295-6c83-43fb-babf-a5aa46ecd422 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:39:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:39:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:48Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c246f225e3d namespace: kserve-ci-e2e-test resourceVersion: "21434" uid: 66d2ed52-abd9-4a70-82c8-5d95bbe5ec27 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:39:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:40:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:39:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:26Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c24dae121e3 namespace: kserve-ci-e2e-test resourceVersion: "21872" uid: d2377fc4-0614-4b54-8052-b18807a94da3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c3474e64d5e namespace: kserve-ci-e2e-test resourceVersion: "22226" uid: 7ae9be0f-02a1-407d-8265-e238385da57c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: 4bc8c230-e449-49d1-b580-5687be1198f6 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-601fd-predictor-646cfc64df-ckjl2.18d35c3474e6c158 namespace: kserve-ci-e2e-test resourceVersion: "22227" uid: f9e1b4ef-930c-4cdb-adb0-647dcef6ce3b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-601fd-predictor-646cfc64df namespace: kserve-ci-e2e-test resourceVersion: "21331" uid: 8507ed5b-250b-40aa-8dfa-d1f34082823a kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: 'Created pod: success-200-isvc-601fd-predictor-646cfc64df-ckjl2' metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-601fd-predictor-646cfc64df.18d35c241ead12a4 namespace: kserve-ci-e2e-test resourceVersion: "21350" uid: 58c9c115-6e46-4814-aa9a-b7527d469a6c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21329" uid: 9f2d09bd-1e6b-4b2a-9c32-8cc2dcd8a236 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Scaled up replica set success-200-isvc-601fd-predictor-646cfc64df from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-601fd-predictor.18d35c2418495629 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: a697c34e-2668-4e16-832b-a84e245a2173 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: a741afe2-4a6b-48ce-9227-8e1f3ea0d84f kind: Event lastTimestamp: "2026-09-08T13:40:17Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:17Z" name: success-200-isvc-601fd-predictor.18d35c2796ba762f namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: bf989c7e-020e-47ed-9758-e1a91513ad1e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: a741afe2-4a6b-48ce-9227-8e1f3ea0d84f kind: Event lastTimestamp: "2026-09-08T13:40:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:17Z" name: success-200-isvc-601fd-predictor.18d35c27973af215 namespace: kserve-ci-e2e-test resourceVersion: "21668" uid: 497aaf5e-3fdf-4244-993b-aa1418372461 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: a741afe2-4a6b-48ce-9227-8e1f3ea0d84f kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: success-200-isvc-601fd-predictor.18d35c2e958404d2 namespace: kserve-ci-e2e-test resourceVersion: "22077" uid: 40a30be6-3d1f-4846-800c-6a7c02b0edc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:40:32Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-601fd-predictor namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: a741afe2-4a6b-48ce-9227-8e1f3ea0d84f kind: Event lastTimestamp: "2026-09-08T13:40:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:40:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:47Z" name: success-200-isvc-601fd-predictor.18d35c2e95f8ed8d namespace: kserve-ci-e2e-test resourceVersion: "22080" uid: a71b6379-5092-447c-b75a-81560241cc72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-601fd namespace: kserve-ci-e2e-test resourceVersion: "21321" uid: e5dec6f0-8788-469a-95d7-7e2d9ba14d95 kind: Event lastTimestamp: "2026-09-08T13:40:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:40:36Z" name: success-200-isvc-601fd.18d35c2418172983 namespace: kserve-ci-e2e-test resourceVersion: "21984" uid: a62d65d4-59f2-41ea-b76f-62c2dea3b3ca reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-601fd namespace: kserve-ci-e2e-test resourceVersion: "21982" uid: e5dec6f0-8788-469a-95d7-7e2d9ba14d95 kind: Event lastTimestamp: "2026-09-08T13:40:36Z" message: InferenceService [success-200-isvc-601fd] is Ready metadata: creationTimestamp: "2026-09-08T13:40:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:40:36Z" name: success-200-isvc-601fd.18d35c2f9659c574 namespace: kserve-ci-e2e-test resourceVersion: "21983" uid: e2da4132-dc6c-4bd1-bd93-441f41c48da0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:40:57.509822Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22260" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-64898-predictor-57884f876f-8wq7r to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c347d8043d2 namespace: kserve-ci-e2e-test resourceVersion: "22266" uid: 4343f10f-f733-47a9-b1c9-47826c43a3a6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-64898-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c348a23795c namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 4b38df9f-6111-4e3c-9e5c-729e615ab23d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22271" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34c3a09a01 namespace: kserve-ci-e2e-test resourceVersion: "22321" uid: 606c88e5-29ef-4573-94ad-b8aa49f0c115 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34c632fcc9 namespace: kserve-ci-e2e-test resourceVersion: "22323" uid: 38b272e0-bc1f-4f03-93c0-2bcb3d775c2c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34c913ed6f namespace: kserve-ci-e2e-test resourceVersion: "22324" uid: b7f644a9-fa32-4b9e-a40f-b3907c8ba1a1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34c9849292 namespace: kserve-ci-e2e-test resourceVersion: "22325" uid: 9fb3a690-2a5a-4e8d-bba6-1d9f881072b0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34c98f0643 namespace: kserve-ci-e2e-test resourceVersion: "22326" uid: 6556cfb4-db9b-4f1c-83b7-cbcc6a24bb03 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34cc88bf2c namespace: kserve-ci-e2e-test resourceVersion: "22328" uid: a8f568fc-716c-4a69-99d3-18bb80b3f8d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:40:58Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:58Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34ccf75e7d namespace: kserve-ci-e2e-test resourceVersion: "22329" uid: b2ec81b5-3f5c-438c-ad21-53130dbe52c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:41:35Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:40:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:35Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35c34fd4e423c namespace: kserve-ci-e2e-test resourceVersion: "22832" uid: bfd0a534-0d10-40b7-ad81-6590cd750054 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35cb5ab0b455b namespace: kserve-ci-e2e-test resourceVersion: "26877" uid: 957702e2-2847-4dec-b8e1-bbc4e4371b55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-64898-predictor-57884f876f-8wq7r namespace: kserve-ci-e2e-test resourceVersion: "22263" uid: 68c6a880-7d55-497a-8b8e-f3f62c7562cd kind: Event lastTimestamp: "2026-09-08T13:50:12Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:12Z" name: success-200-isvc-64898-predictor-57884f876f-8wq7r.18d35cb5ab0bd9a8 namespace: kserve-ci-e2e-test resourceVersion: "26878" uid: 66318783-3b45-42e5-84c3-346123d397e2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-64898-predictor-57884f876f namespace: kserve-ci-e2e-test resourceVersion: "22253" uid: 11d24143-5833-48ce-8025-fb4538d1f841 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: 'Created pod: success-200-isvc-64898-predictor-57884f876f-8wq7r' metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-64898-predictor-57884f876f.18d35c347d10717a namespace: kserve-ci-e2e-test resourceVersion: "22262" uid: bca5b49e-30c8-4cef-beaa-5cd545dac5f8 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22251" uid: 072478da-fcd0-456e-b585-6752afda95f2 kind: Event lastTimestamp: "2026-09-08T13:40:57Z" message: Scaled up replica set success-200-isvc-64898-predictor-57884f876f from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:57Z" name: success-200-isvc-64898-predictor.18d35c347aca7bed namespace: kserve-ci-e2e-test resourceVersion: "22254" uid: a998da07-a464-4d70-9382-27c90e6cf1cc reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: b687376a-9116-4562-8bad-6db4f4ced54c kind: Event lastTimestamp: "2026-09-08T13:42:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:12Z" name: success-200-isvc-64898-predictor.18d35c37f9334126 namespace: kserve-ci-e2e-test resourceVersion: "23193" uid: 8991f5fd-ab78-479d-9b36-202cab7cef8c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:41:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-64898-predictor namespace: kserve-ci-e2e-test resourceVersion: "22252" uid: b687376a-9116-4562-8bad-6db4f4ced54c kind: Event lastTimestamp: "2026-09-08T13:42:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:12Z" name: success-200-isvc-64898-predictor.18d35c37f9ab4439 namespace: kserve-ci-e2e-test resourceVersion: "23194" uid: eb1ca6fa-67e5-44b2-b956-3c97e7758ef7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64898 namespace: kserve-ci-e2e-test resourceVersion: "22240" uid: 265ca51f-7b01-4d7e-9249-48afb555ec87 kind: Event lastTimestamp: "2026-09-08T13:41:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:40:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:45Z" name: success-200-isvc-64898.18d35c347add6889 namespace: kserve-ci-e2e-test resourceVersion: "22939" uid: b38bc13d-16da-4558-afab-42c8e200e2c0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-64898 namespace: kserve-ci-e2e-test resourceVersion: "22937" uid: 265ca51f-7b01-4d7e-9249-48afb555ec87 kind: Event lastTimestamp: "2026-09-08T13:41:45Z" message: InferenceService [success-200-isvc-64898] is Ready metadata: creationTimestamp: "2026-09-08T13:41:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:45Z" name: success-200-isvc-64898.18d35c3fb8e042ba namespace: kserve-ci-e2e-test resourceVersion: "22938" uid: 1a94e291-bb84-4432-80ca-e6815d39c0c7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:51:22.822460Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27804" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7d014-predictor-5688f58dbc-7pbff to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:51:22Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc61509eb78 namespace: kserve-ci-e2e-test resourceVersion: "27809" uid: ea83c016-537c-4f6d-928d-27421ae6d3b2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:23Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7d014-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:23Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc620afde20 namespace: kserve-ci-e2e-test resourceVersion: "27839" uid: 24b81caa-f99c-4504-9c31-63d9f75716f6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:23Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27814" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:23Z" message: Add eth0 [10.133.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:51:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:51:23Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc65a8cade9 namespace: kserve-ci-e2e-test resourceVersion: "27859" uid: 3cb5b49e-8e3d-45c6-b0cf-15e8b04d34c5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc65bfce5d2 namespace: kserve-ci-e2e-test resourceVersion: "27861" uid: 1344c55b-12f3-4062-846f-b968e0f0fa6e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc65efac29f namespace: kserve-ci-e2e-test resourceVersion: "27862" uid: e0f556b9-4572-445d-9eaa-eb3f67e57bcd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc65f6f7cd8 namespace: kserve-ci-e2e-test resourceVersion: "27863" uid: 1b32a7fe-b70f-4f0a-861e-f1ba179e95db reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc65f793ffd namespace: kserve-ci-e2e-test resourceVersion: "27864" uid: 39114bf9-a764-4d4d-b225-857150a0caa9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc662b86174 namespace: kserve-ci-e2e-test resourceVersion: "27868" uid: 459aa7b9-90a8-4e58-9fc0-88ec900ed9f1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:24Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:51:24Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:51:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:24Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc6632f145c namespace: kserve-ci-e2e-test resourceVersion: "27869" uid: 053a2ac6-d6d8-4f38-b74f-0da1942bd78a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:51:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T13:52:01Z" message: 'Readiness probe failed: dial tcp 10.133.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:51:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:52:01Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35cc6a46cef77 namespace: kserve-ci-e2e-test resourceVersion: "28244" uid: 195e1f7e-8193-4f32-9d63-0edd0dafffe5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T14:00:37Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T14:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:00:37Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35d473de58b69 namespace: kserve-ci-e2e-test resourceVersion: "31910" uid: c6d0238a-ae8d-489d-958a-d68e02fb41ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:00:37Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff namespace: kserve-ci-e2e-test resourceVersion: "27806" uid: 6cb32918-a428-4bc0-a5bb-c22184f68e08 kind: Event lastTimestamp: "2026-09-08T14:00:37Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T14:00:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:00:37Z" name: success-200-isvc-7d014-predictor-5688f58dbc-7pbff.18d35d473de5f660 namespace: kserve-ci-e2e-test resourceVersion: "31915" uid: 98eeb860-d535-4811-8a2b-2c8f9f0995ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7d014-predictor-5688f58dbc namespace: kserve-ci-e2e-test resourceVersion: "27798" uid: b68d0928-9f1b-4df5-af01-56a9f3096e67 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: 'Created pod: success-200-isvc-7d014-predictor-5688f58dbc-7pbff' metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:22Z" name: success-200-isvc-7d014-predictor-5688f58dbc.18d35cc6147a35ee namespace: kserve-ci-e2e-test resourceVersion: "27807" uid: a1576318-bdba-4f7a-904d-b22c512d9b69 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27795" uid: 77a5019e-f30c-436f-b8b6-ca3555a27737 kind: Event lastTimestamp: "2026-09-08T13:51:22Z" message: Scaled up replica set success-200-isvc-7d014-predictor-5688f58dbc from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:22Z" name: success-200-isvc-7d014-predictor.18d35cc61190d8f2 namespace: kserve-ci-e2e-test resourceVersion: "27799" uid: bee5fb63-3124-45f9-9033-5613a6738a96 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 7b78d7e1-3f72-4b3b-8917-260dc4c71fd4 kind: Event lastTimestamp: "2026-09-08T13:52:22Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:22Z" name: success-200-isvc-7d014-predictor.18d35cc990081f50 namespace: kserve-ci-e2e-test resourceVersion: "28445" uid: fb55514c-da04-4d4c-a8e2-1362bd5c2f24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:51:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "27797" uid: 7b78d7e1-3f72-4b3b-8917-260dc4c71fd4 kind: Event lastTimestamp: "2026-09-08T13:52:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:22Z" name: success-200-isvc-7d014-predictor.18d35cc9907b95d0 namespace: kserve-ci-e2e-test resourceVersion: "28446" uid: 6dab75fd-67d6-4f4d-93be-39a52d71f292 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "28017" uid: 7b78d7e1-3f72-4b3b-8917-260dc4c71fd4 kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: success-200-isvc-7d014-predictor.18d35cd78c8855e6 namespace: kserve-ci-e2e-test resourceVersion: "28571" uid: 15c046f5-9336-438d-83e8-4a18e4a047f2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7d014-predictor namespace: kserve-ci-e2e-test resourceVersion: "28017" uid: 7b78d7e1-3f72-4b3b-8917-260dc4c71fd4 kind: Event lastTimestamp: "2026-09-08T13:52:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:37Z" name: success-200-isvc-7d014-predictor.18d35cd78d33baf0 namespace: kserve-ci-e2e-test resourceVersion: "28573" uid: 79402a1d-7a88-4287-8f07-ec9be836b0ff reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:51:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d014 namespace: kserve-ci-e2e-test resourceVersion: "27782" uid: a691faf4-b184-461f-b369-492ec5c049f3 kind: Event lastTimestamp: "2026-09-08T13:52:11Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:51:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:52:11Z" name: success-200-isvc-7d014.18d35cc611a0b867 namespace: kserve-ci-e2e-test resourceVersion: "28340" uid: 79167135-02a3-425c-83f1-c739847ca6bd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:11Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7d014 namespace: kserve-ci-e2e-test resourceVersion: "28336" uid: a691faf4-b184-461f-b369-492ec5c049f3 kind: Event lastTimestamp: "2026-09-08T13:52:11Z" message: InferenceService [success-200-isvc-7d014] is Ready metadata: creationTimestamp: "2026-09-08T13:52:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:52:11Z" name: success-200-isvc-7d014.18d35cd1617d7a5c namespace: kserve-ci-e2e-test resourceVersion: "28338" uid: 583e0ef2-40fb-4a45-9ff6-ef039235b297 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:30:32.160276Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16969" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-87be3-predictor-845b844549-cpcg2 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba2e3c36b68 namespace: kserve-ci-e2e-test resourceVersion: "16979" uid: 32e771ee-aa16-4df7-bfcc-49d3c3c9acae reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-87be3-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba2ef9c5d26 namespace: kserve-ci-e2e-test resourceVersion: "17034" uid: d971c76a-1fd3-4b96-b083-60010eaee292 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16990" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba35f5112ca namespace: kserve-ci-e2e-test resourceVersion: "17055" uid: ea08dd33-1522-44f1-8a1e-2a38e1fd61a5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba360b6c132 namespace: kserve-ci-e2e-test resourceVersion: "17057" uid: 1f0c6973-09d6-4a27-bc21-90c2df940c96 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba36407d613 namespace: kserve-ci-e2e-test resourceVersion: "17059" uid: 7eb672df-e271-43a1-ac33-57ebef185af3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba364728dd3 namespace: kserve-ci-e2e-test resourceVersion: "17060" uid: f500dfd6-7428-48b2-b358-5d6d016c5290 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba3647eb2d1 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 03ca5c0e-cbb9-4abf-80eb-4289ec2c26ca reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba3671f6332 namespace: kserve-ci-e2e-test resourceVersion: "17063" uid: 174a35be-f550-4993-97b0-1cfbdf371088 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:30:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:30:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:34Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba367871469 namespace: kserve-ci-e2e-test resourceVersion: "17064" uid: 8d49633c-e518-4f3a-84af-409fb7f19f04 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:30:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:31:11Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:11Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35ba39048dd30 namespace: kserve-ci-e2e-test resourceVersion: "17595" uid: 8ddebacd-6239-4c55-ab80-888c31823cb4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35c2420818f18 namespace: kserve-ci-e2e-test resourceVersion: "21359" uid: a2fa0d83-dbcb-486d-87c7-b85b8abea26f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-87be3-predictor-845b844549-cpcg2 namespace: kserve-ci-e2e-test resourceVersion: "16976" uid: 7cda03f3-9df3-4741-8799-8874f44370b4 kind: Event lastTimestamp: "2026-09-08T13:39:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:39:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:47Z" name: success-200-isvc-87be3-predictor-845b844549-cpcg2.18d35c2420823f65 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: b4d6e6cf-1491-474f-9dda-32a322b257cf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-87be3-predictor-845b844549 namespace: kserve-ci-e2e-test resourceVersion: "16957" uid: 3d22a472-474a-4854-90fe-7a396dbdd0f3 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'Created pod: success-200-isvc-87be3-predictor-845b844549-cpcg2' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3-predictor-845b844549.18d35ba2e34654ea namespace: kserve-ci-e2e-test resourceVersion: "16975" uid: 8839c111-c8d9-4e27-809e-07cd15d49377 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: f404900a-6170-494e-a744-9dc1ad381bfe kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Scaled up replica set success-200-isvc-87be3-predictor-845b844549 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3-predictor.18d35ba2e002cbee namespace: kserve-ci-e2e-test resourceVersion: "16959" uid: 01d9854f-1cf4-42fc-9228-db7bce3fbb57 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: 336fb0aa-857b-4ae1-ac65-fb019ede8fef kind: Event lastTimestamp: "2026-09-08T13:31:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:02Z" name: success-200-isvc-87be3-predictor.18d35ba65ec2f314 namespace: kserve-ci-e2e-test resourceVersion: "17365" uid: bdc02739-3791-445b-8d8c-767e6a5bf8cb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:30:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "16956" uid: 336fb0aa-857b-4ae1-ac65-fb019ede8fef kind: Event lastTimestamp: "2026-09-08T13:31:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:30:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:02Z" name: success-200-isvc-87be3-predictor.18d35ba65f617b4f namespace: kserve-ci-e2e-test resourceVersion: "17366" uid: b12a8b39-b35c-4c09-9784-5f3914699686 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 336fb0aa-857b-4ae1-ac65-fb019ede8fef kind: Event lastTimestamp: "2026-09-08T13:31:47Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:47Z" name: success-200-isvc-87be3-predictor.18d35bad5cca97e6 namespace: kserve-ci-e2e-test resourceVersion: "17955" uid: 063406ac-a086-40bb-9689-55ea17c6433c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:31:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-87be3-predictor namespace: kserve-ci-e2e-test resourceVersion: "17209" uid: 336fb0aa-857b-4ae1-ac65-fb019ede8fef kind: Event lastTimestamp: "2026-09-08T13:31:47Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:31:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:47Z" name: success-200-isvc-87be3-predictor.18d35bad5d49c4fa namespace: kserve-ci-e2e-test resourceVersion: "17956" uid: 44423355-d259-4a56-afd4-4a577b8c5dc2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: 72b2e0f9-cae5-43ad-9ab8-a60b7b95100e kind: Event lastTimestamp: "2026-09-08T13:31:21Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:21Z" name: success-200-isvc-87be3.18d35ba2e0037c59 namespace: kserve-ci-e2e-test resourceVersion: "17707" uid: ce825337-d3e6-4bf1-ad11-baeb0d223894 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: 72b2e0f9-cae5-43ad-9ab8-a60b7b95100e kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'Failed to update status for InferenceService "success-200-isvc-87be3": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87be3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3.18d35ba2e0da697f namespace: kserve-ci-e2e-test resourceVersion: "16963" uid: bd49abc7-afdc-4143-a349-df8d2e9b96a8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "16933" uid: 72b2e0f9-cae5-43ad-9ab8-a60b7b95100e kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-87be3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-87be3.18d35ba2e0dab9a2 namespace: kserve-ci-e2e-test resourceVersion: "16965" uid: 4ed44b17-f884-499a-9643-1077a5cb2f4c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:21Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17705" uid: 72b2e0f9-cae5-43ad-9ab8-a60b7b95100e kind: Event lastTimestamp: "2026-09-08T13:31:21Z" message: InferenceService [success-200-isvc-87be3] is Ready metadata: creationTimestamp: "2026-09-08T13:31:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:21Z" name: success-200-isvc-87be3.18d35bae4b87dd3b namespace: kserve-ci-e2e-test resourceVersion: "17706" uid: 7c24e736-7a6d-460b-9c8e-c5fab947696b reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:41:33.326963Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22749" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c7da0-predictor-945f88485-8srks to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cd45e6436 namespace: kserve-ci-e2e-test resourceVersion: "22754" uid: feb84d7e-a3a3-45ab-8390-a4727c578e0e reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22759" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cf660bb69 namespace: kserve-ci-e2e-test resourceVersion: "22792" uid: 06d65966-5e8a-439d-853e-095185477522 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cf819b3f9 namespace: kserve-ci-e2e-test resourceVersion: "22794" uid: 8086ca9b-f796-481d-ae5f-f0d283207bf3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cfb5775c8 namespace: kserve-ci-e2e-test resourceVersion: "22797" uid: de5fbe8d-3aa2-44b7-884a-4c1f03c4b6d1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cfbcbdd08 namespace: kserve-ci-e2e-test resourceVersion: "22798" uid: eb5914b0-d3c2-437c-a1cb-10ce0a96ed9b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cfbd61aa4 namespace: kserve-ci-e2e-test resourceVersion: "22799" uid: 593dc2c2-4781-4797-88fe-c5eb10fcc69a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cfe8eb795 namespace: kserve-ci-e2e-test resourceVersion: "22801" uid: 3d6a8946-fa95-4100-a30e-aadb0d6390bb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:41:34Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:34Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3cfefcc448 namespace: kserve-ci-e2e-test resourceVersion: "22802" uid: 5435358f-81f1-4d90-8696-5a6e6191bba0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-08T13:41:34Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:50:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:41:34Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:50Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35c3d2d2bb305 namespace: kserve-ci-e2e-test resourceVersion: "27413" uid: 345c671f-0da0-4739-9860-5d40271251e1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35cbdef8a63f2 namespace: kserve-ci-e2e-test resourceVersion: "27286" uid: ae6bee6a-d607-4b35-9818-c843a1c9fb89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35cbdef8afe63 namespace: kserve-ci-e2e-test resourceVersion: "27288" uid: 9879a00f-48e4-4466-a30a-e30cd76dcf96 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c7da0-predictor-945f88485-8srks namespace: kserve-ci-e2e-test resourceVersion: "22751" uid: 4f0f9568-8937-4421-a904-2714533d3982 kind: Event lastTimestamp: "2026-09-08T13:50:50Z" message: 'Readiness probe failed: Get "https://10.133.0.32:8643/healthz": dial tcp 10.133.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:50Z" name: success-200-isvc-c7da0-predictor-945f88485-8srks.18d35cbea1926b18 namespace: kserve-ci-e2e-test resourceVersion: "27411" uid: e9219213-6958-4187-b46e-ab568c834b11 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c7da0-predictor-945f88485 namespace: kserve-ci-e2e-test resourceVersion: "22743" uid: d03cbcfd-08db-4499-84df-8f463d47893b kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: 'Created pod: success-200-isvc-c7da0-predictor-945f88485-8srks' metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor-945f88485.18d35c3cd3cefb92 namespace: kserve-ci-e2e-test resourceVersion: "22752" uid: 50e62361-a234-48d1-bda8-96ca7f829e84 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22741" uid: 5dfa0307-f2d3-47f2-969b-830084b0b0d8 kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Scaled up replica set success-200-isvc-c7da0-predictor-945f88485 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-c7da0-predictor.18d35c3cd0dfbbac namespace: kserve-ci-e2e-test resourceVersion: "22745" uid: 6c678ff0-1f98-4eb4-9f28-2e0fd771c9be reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22742" uid: f77cf128-196e-4813-a5dc-d56e14eced7d kind: Event lastTimestamp: "2026-09-08T13:42:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:03Z" name: success-200-isvc-c7da0-predictor.18d35c404f58b5a9 namespace: kserve-ci-e2e-test resourceVersion: "23117" uid: 4552903c-3c2c-4b9a-8ad0-5dd73111df8d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:41:48Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22742" uid: f77cf128-196e-4813-a5dc-d56e14eced7d kind: Event lastTimestamp: "2026-09-08T13:42:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:41:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:03Z" name: success-200-isvc-c7da0-predictor.18d35c404fd0e800 namespace: kserve-ci-e2e-test resourceVersion: "23118" uid: bb1f7f02-ac06-475f-b0ea-8333e89d7849 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22958" uid: f77cf128-196e-4813-a5dc-d56e14eced7d kind: Event lastTimestamp: "2026-09-08T13:42:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:48Z" name: success-200-isvc-c7da0-predictor.18d35c474dcd2f9f namespace: kserve-ci-e2e-test resourceVersion: "23519" uid: e011a931-d953-4f0e-ad41-7070e72edb88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:42:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c7da0-predictor namespace: kserve-ci-e2e-test resourceVersion: "22958" uid: f77cf128-196e-4813-a5dc-d56e14eced7d kind: Event lastTimestamp: "2026-09-08T13:42:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:42:48Z" name: success-200-isvc-c7da0-predictor.18d35c474e415682 namespace: kserve-ci-e2e-test resourceVersion: "23520" uid: 99ca7a53-061a-4764-a101-954925a2a5e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "22729" uid: 31a287b7-9420-4494-a769-dddb1eb122a3 kind: Event lastTimestamp: "2026-09-08T13:42:20Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:42:20Z" name: success-200-isvc-c7da0.18d35c3cd0e30ff0 namespace: kserve-ci-e2e-test resourceVersion: "23274" uid: afbcce0b-0dbd-4866-bcb3-46a8c2406336 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:42:20Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c7da0 namespace: kserve-ci-e2e-test resourceVersion: "23272" uid: 31a287b7-9420-4494-a769-dddb1eb122a3 kind: Event lastTimestamp: "2026-09-08T13:42:20Z" message: InferenceService [success-200-isvc-c7da0] is Ready metadata: creationTimestamp: "2026-09-08T13:42:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:42:20Z" name: success-200-isvc-c7da0.18d35c47ea036caa namespace: kserve-ci-e2e-test resourceVersion: "23273" uid: c18063e2-735b-49bf-8c70-1e605ef97aec reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:50:47.918310Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27309" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-ce1f6-predictor-76644689b9-smjt7 to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbdf496d54e namespace: kserve-ci-e2e-test resourceVersion: "27317" uid: b9796a11-10d5-4058-8241-8db85fe64014 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-ce1f6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:50:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:48Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe00ab32d1 namespace: kserve-ci-e2e-test resourceVersion: "27353" uid: d67bfb9f-bd52-4818-92fd-5b37bee66220 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27327" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe3b00285f namespace: kserve-ci-e2e-test resourceVersion: "27377" uid: 68699a1a-46ad-45cf-8c72-9a8c9860d223 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe3ccbe107 namespace: kserve-ci-e2e-test resourceVersion: "27379" uid: b0a664ed-413b-49f9-8cd2-24d3f75614c3 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe407c7357 namespace: kserve-ci-e2e-test resourceVersion: "27380" uid: 89dd72ac-9c8f-4b73-a4a5-f1b5aff5e980 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe40f12759 namespace: kserve-ci-e2e-test resourceVersion: "27381" uid: 78f7506b-6594-4bc4-bb97-0274531208d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe40fbb4e9 namespace: kserve-ci-e2e-test resourceVersion: "27382" uid: 30f21978-b633-4d90-873f-6b63a3b92941 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe43f6dcb0 namespace: kserve-ci-e2e-test resourceVersion: "27383" uid: f296f182-cc12-41e6-b2e9-1a6c18835fbd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:50:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:50:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:50:49Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbe446deb28 namespace: kserve-ci-e2e-test resourceVersion: "27385" uid: 2b2fffa3-29a5-444e-bd05-42bf35aeba25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:50:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T13:51:27Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:50:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:27Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35cbeb1344dbb namespace: kserve-ci-e2e-test resourceVersion: "27917" uid: 82604926-39eb-41ba-8a96-ad228d842b7a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T14:08:07Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T14:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:07Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35dafff008ed2 namespace: kserve-ci-e2e-test resourceVersion: "35021" uid: 96a507d5-5259-453a-9174-6f4baace43bc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T14:08:07Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T14:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:07Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35dafff00ff75 namespace: kserve-ci-e2e-test resourceVersion: "35027" uid: 35bdc687-e76a-418c-9dc7-bfb3080966d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7 namespace: kserve-ci-e2e-test resourceVersion: "27313" uid: 6077a8b1-0ed9-4e61-ba96-09329e8af889 kind: Event lastTimestamp: "2026-09-08T14:08:12Z" message: 'Readiness probe failed: Get "https://10.133.0.34:8643/healthz": dial tcp 10.133.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-08T14:08:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:12Z" name: success-200-isvc-ce1f6-predictor-76644689b9-smjt7.18d35db111f56783 namespace: kserve-ci-e2e-test resourceVersion: "35106" uid: c22d0e69-1ff6-4ea4-8557-40cedd15fc5d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-ce1f6-predictor-76644689b9 namespace: kserve-ci-e2e-test resourceVersion: "27295" uid: 7b1131fe-4104-414e-93eb-f5375a8d0be6 kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: 'Created pod: success-200-isvc-ce1f6-predictor-76644689b9-smjt7' metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-ce1f6-predictor-76644689b9.18d35cbdf4192d36 namespace: kserve-ci-e2e-test resourceVersion: "27311" uid: f74988c0-2408-433b-82d6-ea040c8920a6 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27291" uid: cbcaaa94-d982-4292-8622-bff173d65eb6 kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: Scaled up replica set success-200-isvc-ce1f6-predictor-76644689b9 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-ce1f6-predictor.18d35cbdf190c3fc namespace: kserve-ci-e2e-test resourceVersion: "27296" uid: f91cfd14-5821-40f7-adba-fdd161b57669 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27294" uid: 90c3c388-a533-43c2-9dbd-dbb7dbe496a9 kind: Event lastTimestamp: "2026-09-08T13:51:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:02Z" name: success-200-isvc-ce1f6-predictor.18d35cc170119497 namespace: kserve-ci-e2e-test resourceVersion: "27536" uid: 5b116f40-0a0d-42c1-b3a8-8e928621931a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27294" uid: 90c3c388-a533-43c2-9dbd-dbb7dbe496a9 kind: Event lastTimestamp: "2026-09-08T13:51:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:51:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:02Z" name: success-200-isvc-ce1f6-predictor.18d35cc17088111d namespace: kserve-ci-e2e-test resourceVersion: "27538" uid: 4cf5ce82-d4d1-49c5-ba1d-6ef49148f961 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:51:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: 90c3c388-a533-43c2-9dbd-dbb7dbe496a9 kind: Event lastTimestamp: "2026-09-08T13:52:02Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:02Z" name: success-200-isvc-ce1f6-predictor.18d35cc4efb7dedf namespace: kserve-ci-e2e-test resourceVersion: "28260" uid: 8e107aec-95da-45ef-b317-de83d68421bd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-08T13:51:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-ce1f6-predictor namespace: kserve-ci-e2e-test resourceVersion: "27537" uid: 90c3c388-a533-43c2-9dbd-dbb7dbe496a9 kind: Event lastTimestamp: "2026-09-08T13:52:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:51:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:02Z" name: success-200-isvc-ce1f6-predictor.18d35cc4f0526f01 namespace: kserve-ci-e2e-test resourceVersion: "28261" uid: 2b88fdbb-408b-4283-b21e-04d079827d43 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: bc04d2db-b1f4-417d-85f1-e09c44b4b87a kind: Event lastTimestamp: "2026-09-08T13:51:37Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:37Z" name: success-200-isvc-ce1f6.18d35cbdf1909d64 namespace: kserve-ci-e2e-test resourceVersion: "28010" uid: f498d16a-3818-4742-ac7e-e350f0719ca7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: bc04d2db-b1f4-417d-85f1-e09c44b4b87a kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: 'Failed to update status for InferenceService "success-200-isvc-ce1f6": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ce1f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-ce1f6.18d35cbdf2839f23 namespace: kserve-ci-e2e-test resourceVersion: "27300" uid: dd1331c9-f5ff-4552-8908-c3a3b92ea250 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:50:47Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "27270" uid: bc04d2db-b1f4-417d-85f1-e09c44b4b87a kind: Event lastTimestamp: "2026-09-08T13:50:47Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-ce1f6": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:50:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:50:47Z" name: success-200-isvc-ce1f6.18d35cbdf284786e namespace: kserve-ci-e2e-test resourceVersion: "27302" uid: 075af0bd-60de-42b2-b4d6-98b0fff5d072 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:37Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28007" uid: bc04d2db-b1f4-417d-85f1-e09c44b4b87a kind: Event lastTimestamp: "2026-09-08T13:51:37Z" message: InferenceService [success-200-isvc-ce1f6] is Ready metadata: creationTimestamp: "2026-09-08T13:51:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:37Z" name: success-200-isvc-ce1f6.18d35cc96d1da3fe namespace: kserve-ci-e2e-test resourceVersion: "28008" uid: 6ea8612b-ec9b-4fe6-9ed3-506b3257500f reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:40:23.061331Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21781" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:40:23Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2c7835f55b namespace: kserve-ci-e2e-test resourceVersion: "21786" uid: 16698ad4-f4e5-4bbf-890f-503f776de39a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:24Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : failed to sync secret cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-08T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:24Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cbd813f75 namespace: kserve-ci-e2e-test resourceVersion: "21838" uid: 68b2187e-a1f7-47db-b8d0-a0e38466d02d reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:24Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:24Z" message: 'MountVolume.SetUp failed for volume "kube-rbac-proxy-sar-config" : failed to sync configmap cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-08T13:40:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:24Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cbd838454 namespace: kserve-ci-e2e-test resourceVersion: "21839" uid: 2c056f4f-5d33-4190-b9d7-a1a65078af1a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21788" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cf3c55f89 namespace: kserve-ci-e2e-test resourceVersion: "21852" uid: e4a817c5-92fd-4845-9101-96ec7ddcacaa reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cf55a74b2 namespace: kserve-ci-e2e-test resourceVersion: "21854" uid: c0ecbe87-edbb-4e55-8fbf-643f91961c97 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cf820c5d8 namespace: kserve-ci-e2e-test resourceVersion: "21855" uid: 8aa3fb72-e82f-46cf-8a48-862d3bcce0f6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cf89b52e0 namespace: kserve-ci-e2e-test resourceVersion: "21856" uid: 284fecdd-caa0-4b6b-b04b-fe79767f3e11 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cf8a6523a namespace: kserve-ci-e2e-test resourceVersion: "21857" uid: 6935a490-19a6-4815-ae7c-ad34de9ca9d7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cfba58c2e namespace: kserve-ci-e2e-test resourceVersion: "21858" uid: 3775845b-39c2-43b3-a9db-ca1be6f8f31e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:25Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:40:25Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:40:25Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:25Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2cfc3b7b59 namespace: kserve-ci-e2e-test resourceVersion: "21859" uid: 829854a7-cca8-432b-a8e7-5d788e19219a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:41:02Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:40:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:02Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c2d4649a36e namespace: kserve-ci-e2e-test resourceVersion: "22374" uid: 99ddbb1c-9540-4058-8a5a-69c2a0e8ef48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c3cca56585a namespace: kserve-ci-e2e-test resourceVersion: "22719" uid: 9752bae6-8e63-4d59-a90e-87d166ff22fb reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:33Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: e5d5a9db-31ea-4c49-925e-5910db3e30da kind: Event lastTimestamp: "2026-09-08T13:41:33Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:41:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:41:33Z" name: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z.18d35c3cca56e730 namespace: kserve-ci-e2e-test resourceVersion: "22723" uid: 924ccc28-8351-4c6d-ac64-95be856a3d09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:23Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e90df-predictor-5bbcb468bc namespace: kserve-ci-e2e-test resourceVersion: "21774" uid: eb0f1bb9-cfa8-469e-ae43-88db95282b1b kind: Event lastTimestamp: "2026-09-08T13:40:23Z" message: 'Created pod: success-200-isvc-e90df-predictor-5bbcb468bc-wkj7z' metadata: creationTimestamp: "2026-09-08T13:40:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:23Z" name: success-200-isvc-e90df-predictor-5bbcb468bc.18d35c2c76b6d9f5 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: 81b4d04e-82e7-4c90-8c40-90e8da9f021e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21772" uid: f799d92a-2ba2-4cfb-8373-e4f9a74dad26 kind: Event lastTimestamp: "2026-09-08T13:40:22Z" message: Scaled up replica set success-200-isvc-e90df-predictor-5bbcb468bc from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:40:22Z" name: success-200-isvc-e90df-predictor.18d35c2c725c7192 namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: b59441ee-0724-4490-9405-43dec349fd77 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:40:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21773" uid: 82865a3e-7b89-40c3-9643-9214ab0a83ca kind: Event lastTimestamp: "2026-09-08T13:41:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:08Z" name: success-200-isvc-e90df-predictor.18d35c2ff161890a namespace: kserve-ci-e2e-test resourceVersion: "22414" uid: a0717d87-ba31-4adc-ae0e-3504e13f2ab6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:40:37Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21773" uid: 82865a3e-7b89-40c3-9643-9214ab0a83ca kind: Event lastTimestamp: "2026-09-08T13:41:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:40:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:08Z" name: success-200-isvc-e90df-predictor.18d35c2ff1d4bded namespace: kserve-ci-e2e-test resourceVersion: "22415" uid: b797e4e9-b6e5-41db-b173-c6d40f41a09c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 82865a3e-7b89-40c3-9643-9214ab0a83ca kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:23Z" name: success-200-isvc-e90df-predictor.18d35c3a6f440803 namespace: kserve-ci-e2e-test resourceVersion: "22578" uid: 9bf934b6-63b2-4d9f-9e77-fe7450954bd3 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e90df-predictor namespace: kserve-ci-e2e-test resourceVersion: "21997" uid: 82865a3e-7b89-40c3-9643-9214ab0a83ca kind: Event lastTimestamp: "2026-09-08T13:41:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:41:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:41:23Z" name: success-200-isvc-e90df-predictor.18d35c3a6fae0f97 namespace: kserve-ci-e2e-test resourceVersion: "22581" uid: 44561572-251b-45a4-ab9f-0d232773a1e7 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:40:22Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e90df namespace: kserve-ci-e2e-test resourceVersion: "21759" uid: e3638e36-e019-460a-858a-216a5ce6a715 kind: Event lastTimestamp: "2026-09-08T13:41:12Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:40:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:12Z" name: success-200-isvc-e90df.18d35c2c7277d798 namespace: kserve-ci-e2e-test resourceVersion: "22476" uid: 51f33fc3-29b8-4fc9-b095-4b678c5cd750 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:41:12Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e90df namespace: kserve-ci-e2e-test resourceVersion: "22473" uid: e3638e36-e019-460a-858a-216a5ce6a715 kind: Event lastTimestamp: "2026-09-08T13:41:12Z" message: InferenceService [success-200-isvc-e90df] is Ready metadata: creationTimestamp: "2026-09-08T13:41:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:41:12Z" name: success-200-isvc-e90df.18d35c3802ec177b namespace: kserve-ci-e2e-test resourceVersion: "22474" uid: 2965d2ea-a939-45c3-88fe-266d25528a4a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:28:57.779873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15857" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-f794f-predictor-75b57f85bc-4s5nf to ip-10-0-140-239.ec2.internal metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:28:57Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8cea40c167 namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 4abccaa9-1ec0-4bbd-8409-4cb713757b5f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:28:57Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-f794f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:57Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8cf2a2bfaf namespace: kserve-ci-e2e-test resourceVersion: "15877" uid: c2d4ecf5-d8f6-4ae5-b027-769756ec6fdc reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15867" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:28:58Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8d2ffe284e namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: d78c317b-33aa-40a1-8fd5-eccc476cbfe0 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:28:58Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" metadata: creationTimestamp: "2026-09-08T13:28:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:28:58Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8d31b90d68 namespace: kserve-ci-e2e-test resourceVersion: "16008" uid: dc9ee42d-8e18-4e4b-aa8f-70ef4b8ee025 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:10Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1946" in 11.693s (11.693s including waiting). Image size: 1356695628 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:10Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8feabc5130 namespace: kserve-ci-e2e-test resourceVersion: "16139" uid: ffe652a6-06f6-4e1e-8e05-740bf9d97940 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:10Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-08T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:10Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8feec02c75 namespace: kserve-ci-e2e-test resourceVersion: "16142" uid: 83fb8a2c-e1b6-4f63-b895-f3c4e64345b6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:10Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-08T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:10Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8fef774b83 namespace: kserve-ci-e2e-test resourceVersion: "16143" uid: cfd3903b-9fa8-4c20-8c63-76f8e340a651 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:10Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-08T13:29:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:10Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b8fef8d2180 namespace: kserve-ci-e2e-test resourceVersion: "16144" uid: 480745bf-5b50-429b-960e-86d5510376ab reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 8.957s (8.957s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b920579687f namespace: kserve-ci-e2e-test resourceVersion: "16237" uid: 4ee6745b-85a0-4664-bf00-ed0f86e4c20b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b920b683da3 namespace: kserve-ci-e2e-test resourceVersion: "16239" uid: 3d4f004e-76b9-4bda-a016-1b343a65e5a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:29:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:19Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:29:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:19Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b920d498dda namespace: kserve-ci-e2e-test resourceVersion: "16240" uid: cd3d4060-357f-4f92-bf77-b66e93881ef1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:29:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:29:56Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-08T13:29:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:29:56Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35b9241f7b98d namespace: kserve-ci-e2e-test resourceVersion: "16569" uid: ca27f590-ddfd-49ad-b95a-039d622213ee reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35ba2decc0397 namespace: kserve-ci-e2e-test resourceVersion: "16951" uid: ef14a4fd-8beb-48ba-82fe-169a5a6b3720 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf namespace: kserve-ci-e2e-test resourceVersion: "15860" uid: ca4543f3-e757-4b16-978f-862868529523 kind: Event lastTimestamp: "2026-09-08T13:30:32Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-08T13:30:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:32Z" name: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf.18d35ba2decc70a8 namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: b99a759e-0ebf-496f-9d23-cce657404a18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-239.ec2.internal source: component: kubelet host: ip-10-0-140-239.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-f794f-predictor-75b57f85bc namespace: kserve-ci-e2e-test resourceVersion: "15847" uid: 39431c50-b683-4f24-a065-ad5a486bb218 kind: Event lastTimestamp: "2026-09-08T13:28:57Z" message: 'Created pod: success-200-isvc-f794f-predictor-75b57f85bc-4s5nf' metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:57Z" name: success-200-isvc-f794f-predictor-75b57f85bc.18d35b8ce97ec939 namespace: kserve-ci-e2e-test resourceVersion: "15858" uid: fde97046-1278-4977-b266-60c0396164ef reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15846" uid: 3f030453-666b-436d-be30-920916f06dc7 kind: Event lastTimestamp: "2026-09-08T13:28:57Z" message: Scaled up replica set success-200-isvc-f794f-predictor-75b57f85bc from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:28:57Z" name: success-200-isvc-f794f-predictor.18d35b8ce2090645 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: a332668b-73d5-499e-843c-1744fd8f1f34 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 31b8f1b3-39cb-4187-8ff1-cac054d559ce kind: Event lastTimestamp: "2026-09-08T13:29:42Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:42Z" name: success-200-isvc-f794f-predictor.18d35b9067e8edab namespace: kserve-ci-e2e-test resourceVersion: "16458" uid: f09dd019-0432-42de-b7a2-61987c22c102 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "15854" uid: 31b8f1b3-39cb-4187-8ff1-cac054d559ce kind: Event lastTimestamp: "2026-09-08T13:29:42Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:29:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:29:42Z" name: success-200-isvc-f794f-predictor.18d35b90683fdfc0 namespace: kserve-ci-e2e-test resourceVersion: "16459" uid: 3d2df068-391f-4f21-bf22-8b71673ee254 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 31b8f1b3-39cb-4187-8ff1-cac054d559ce kind: Event lastTimestamp: "2026-09-08T13:30:27Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:27Z" name: success-200-isvc-f794f-predictor.18d35b9ae4e5ce7e namespace: kserve-ci-e2e-test resourceVersion: "16867" uid: 73d90662-151c-4d9f-bc23-c3ffddc788e5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-08T13:29:57Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-f794f-predictor namespace: kserve-ci-e2e-test resourceVersion: "16159" uid: 31b8f1b3-39cb-4187-8ff1-cac054d559ce kind: Event lastTimestamp: "2026-09-08T13:30:27Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:29:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:27Z" name: success-200-isvc-f794f-predictor.18d35b9ae55c1c00 namespace: kserve-ci-e2e-test resourceVersion: "16868" uid: 3ded3f85-337f-48d8-967d-31a6090d5007 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T13:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f794f namespace: kserve-ci-e2e-test resourceVersion: "15839" uid: a6f12c90-00ed-4154-8c06-28d3c54f49cf kind: Event lastTimestamp: "2026-09-08T13:30:06Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-08T13:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:06Z" name: success-200-isvc-f794f.18d35b8ce81e9aa2 namespace: kserve-ci-e2e-test resourceVersion: "16674" uid: 04232bde-875f-49f7-be33-3bb706ea5e08 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-f794f namespace: kserve-ci-e2e-test resourceVersion: "16670" uid: a6f12c90-00ed-4154-8c06-28d3c54f49cf kind: Event lastTimestamp: "2026-09-08T13:30:06Z" message: InferenceService [success-200-isvc-f794f] is Ready metadata: creationTimestamp: "2026-09-08T13:30:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:06Z" name: success-200-isvc-f794f.18d35b9cfdb78b50 namespace: kserve-ci-e2e-test resourceVersion: "16672" uid: 7929c05b-f29c-4d15-bf49-8f8038b7c2a7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:31:32.184786Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17806" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-87be3-bdb646768-z4lgk to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35bb0dd80c422 namespace: kserve-ci-e2e-test resourceVersion: "17811" uid: 54d926cd-f408-44cc-a199-97a327c09311 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17815" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: Add eth0 [10.132.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35bb0ff6df561 namespace: kserve-ci-e2e-test resourceVersion: "17830" uid: 7fba0f6f-2cd1-4e5e-ae24-d24cbe729c3a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87be3} kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35bb100a69c67 namespace: kserve-ci-e2e-test resourceVersion: "17833" uid: 3b6b25b2-d8d5-4f27-bb2b-3d898c096903 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87be3} kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: 'Created container: switch-graph-87be3' metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35bb10319f2e4 namespace: kserve-ci-e2e-test resourceVersion: "17834" uid: 035eee9a-1a38-4a52-a981-2b2527b72c54 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87be3} kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: Started container switch-graph-87be3 metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35bb103a6e1c6 namespace: kserve-ci-e2e-test resourceVersion: "17835" uid: 37722f01-d345-450b-8782-e81ee1b0ec16 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:39:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87be3} kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:39:46Z" message: Stopping container switch-graph-87be3 metadata: creationTimestamp: "2026-09-08T13:39:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:39:46Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35c24095cfa56 namespace: kserve-ci-e2e-test resourceVersion: "21313" uid: 4be4ea3f-b2ba-4c80-8064-e3256dff75e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:39:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-87be3} kind: Pod name: switch-graph-87be3-bdb646768-z4lgk namespace: kserve-ci-e2e-test resourceVersion: "17808" uid: 73470095-ecbc-4f59-8821-21cd9ec14413 kind: Event lastTimestamp: "2026-09-08T13:40:14Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:39:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:40:14Z" name: switch-graph-87be3-bdb646768-z4lgk.18d35c249bcd088f namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 31851cc7-2c09-4230-aebe-238acef992f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-87be3-bdb646768 namespace: kserve-ci-e2e-test resourceVersion: "17801" uid: 8793f541-ee8f-4971-a384-53502694004b kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: 'Created pod: switch-graph-87be3-bdb646768-z4lgk' metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3-bdb646768.18d35bb0dd15c988 namespace: kserve-ci-e2e-test resourceVersion: "17809" uid: 80f84cda-95dc-4135-94c8-91bdd217d4a4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17799" uid: 6b692e48-2186-4d9d-8320-c42bd642c1b0 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: Scaled up replica set switch-graph-87be3-bdb646768 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3.18d35bb0dc25afbd namespace: kserve-ci-e2e-test resourceVersion: "17803" uid: 6e486566-50b3-4614-a9b5-fe9a42c30c17 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17817" uid: a6e1cb9a-ba7a-4569-b74b-8aca1e8fba78 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: InferenceGraph [switch-graph-87be3] is Ready metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3.18d35bb0e1dc8e8b namespace: kserve-ci-e2e-test resourceVersion: "17819" uid: 21e73d82-9faa-4d8e-b844-a6635e2f0bbd reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: a6e1cb9a-ba7a-4569-b74b-8aca1e8fba78 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: 'Failed to update status for InferenceGraph "switch-graph-87be3": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-87be3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3.18d35bb0e45cc2fe namespace: kserve-ci-e2e-test resourceVersion: "17823" uid: 859d23f6-0d48-4f09-9d7b-9266ad7b5d2f reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:31:32Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17794" uid: a6e1cb9a-ba7a-4569-b74b-8aca1e8fba78 kind: Event lastTimestamp: "2026-09-08T13:31:32Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-87be3": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:31:32Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:31:32Z" name: switch-graph-87be3.18d35bb0e45d0ef1 namespace: kserve-ci-e2e-test resourceVersion: "17824" uid: f3496611-63df-45b0-80ed-fd9301d1eaf1 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:31:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17800" uid: 30d863a0-ef5a-4376-8393-5e21f8e20f8c kind: Event lastTimestamp: "2026-09-08T13:32:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:02Z" name: switch-graph-87be3.18d35bb45aaa6380 namespace: kserve-ci-e2e-test resourceVersion: "18086" uid: 5749bd3a-55e1-413f-8fb4-caa987ecdaef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-08T13:31:47Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-87be3 namespace: kserve-ci-e2e-test resourceVersion: "17800" uid: 30d863a0-ef5a-4376-8393-5e21f8e20f8c kind: Event lastTimestamp: "2026-09-08T13:32:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:31:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:32:02Z" name: switch-graph-87be3.18d35bb45b127cf4 namespace: kserve-ci-e2e-test resourceVersion: "18087" uid: fd19d58e-eb78-47af-bfc4-0fb3dc570552 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:51:47.960363Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28106" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-ce1f6-75dbfb64bd-q4pxj to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:51:47Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:51:47Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35ccbef5fd079 namespace: kserve-ci-e2e-test resourceVersion: "28114" uid: 25493771-8b08-4cc4-b477-1478508e5d36 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28118" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:51:48Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35ccc112007d2 namespace: kserve-ci-e2e-test resourceVersion: "28128" uid: cc02d817-5ade-4288-8f88-3197fa14a8db reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce1f6} kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" already present on machine metadata: creationTimestamp: "2026-09-08T13:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:48Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35ccc12ac835f namespace: kserve-ci-e2e-test resourceVersion: "28131" uid: e5902252-e3e0-4536-9d60-5bbaf8d40ea0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce1f6} kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: 'Created container: switch-graph-ce1f6' metadata: creationTimestamp: "2026-09-08T13:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:48Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35ccc1574788f namespace: kserve-ci-e2e-test resourceVersion: "28132" uid: 8c8b9915-5acc-44d1-9581-d1caed8c3091 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce1f6} kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: Started container switch-graph-ce1f6 metadata: creationTimestamp: "2026-09-08T13:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:51:48Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35ccc15e797c7 namespace: kserve-ci-e2e-test resourceVersion: "28133" uid: 7c6339bc-17df-4f29-9cd4-f571a5bba84d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T14:08:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce1f6} kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T14:08:07Z" message: Stopping container switch-graph-ce1f6 metadata: creationTimestamp: "2026-09-08T14:08:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:07Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35daff61a1f8b namespace: kserve-ci-e2e-test resourceVersion: "35004" uid: ca06223c-6a61-43ee-a6da-1700aee33fb5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-08T14:08:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-ce1f6} kind: Pod name: switch-graph-ce1f6-75dbfb64bd-q4pxj namespace: kserve-ci-e2e-test resourceVersion: "28108" uid: 4297ef51-ddc1-4f75-bc84-1dd4e804854b kind: Event lastTimestamp: "2026-09-08T14:08:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T14:08:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T14:08:30Z" name: switch-graph-ce1f6-75dbfb64bd-q4pxj.18d35db0b506f629 namespace: kserve-ci-e2e-test resourceVersion: "35248" uid: 123bc83a-3e92-45a7-b418-dc7b0c3937a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:47Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-ce1f6-75dbfb64bd namespace: kserve-ci-e2e-test resourceVersion: "28103" uid: 8b37de87-ffa4-49a1-ab67-a39710d955e6 kind: Event lastTimestamp: "2026-09-08T13:51:47Z" message: 'Created pod: switch-graph-ce1f6-75dbfb64bd-q4pxj' metadata: creationTimestamp: "2026-09-08T13:51:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:47Z" name: switch-graph-ce1f6-75dbfb64bd.18d35ccbee961ddb namespace: kserve-ci-e2e-test resourceVersion: "28110" uid: 9920c281-bf43-4a5f-92e3-8a4c0463c884 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:47Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28101" uid: 83c26ce6-c7e4-4867-9316-3f59bbba3b4b kind: Event lastTimestamp: "2026-09-08T13:51:47Z" message: Scaled up replica set switch-graph-ce1f6-75dbfb64bd from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:51:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:51:47Z" name: switch-graph-ce1f6.18d35ccbed7747f0 namespace: kserve-ci-e2e-test resourceVersion: "28104" uid: 7b1acf41-aa1d-4e86-b17d-f036160dc948 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:51:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28121" uid: ff9d55c7-f7fd-4bae-925b-962a18de7fea kind: Event lastTimestamp: "2026-09-08T13:51:48Z" message: InferenceGraph [switch-graph-ce1f6] is Ready metadata: creationTimestamp: "2026-09-08T13:51:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:51:48Z" name: switch-graph-ce1f6.18d35ccbf4636610 namespace: kserve-ci-e2e-test resourceVersion: "28123" uid: 3e9a30dc-1108-4ec8-9475-3151f4660646 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28102" uid: 1668cc7d-c4b5-4894-b0e0-81520050a821 kind: Event lastTimestamp: "2026-09-08T13:52:02Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:52:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:02Z" name: switch-graph-ce1f6.18d35ccf6be0fd7b namespace: kserve-ci-e2e-test resourceVersion: "28258" uid: 8e6c0337-911e-416e-a925-cd7f3cfe59dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:02Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28102" uid: 1668cc7d-c4b5-4894-b0e0-81520050a821 kind: Event lastTimestamp: "2026-09-08T13:52:02Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-08T13:52:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:02Z" name: switch-graph-ce1f6.18d35ccf6c4beb7c namespace: kserve-ci-e2e-test resourceVersion: "28259" uid: 5991a878-d9c8-46bd-8ffa-6d8145283af2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28257" uid: 1668cc7d-c4b5-4894-b0e0-81520050a821 kind: Event lastTimestamp: "2026-09-08T13:52:17Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:17Z" name: switch-graph-ce1f6.18d35cd2eb234d41 namespace: kserve-ci-e2e-test resourceVersion: "28390" uid: 32a4152f-4d4a-4519-9750-c14a52d176f4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:52:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-ce1f6 namespace: kserve-ci-e2e-test resourceVersion: "28257" uid: 1668cc7d-c4b5-4894-b0e0-81520050a821 kind: Event lastTimestamp: "2026-09-08T13:52:17Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-08T13:52:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:52:17Z" name: switch-graph-ce1f6.18d35cd2ebb12e58 namespace: kserve-ci-e2e-test resourceVersion: "28392" uid: 8c7e7917-fed2-4e3d-83e6-81898040578f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-08T13:30:17.788312Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16769" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-f794f-6766495c86-m57t4 to ip-10-0-133-71.ec2.internal metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f-6766495c86-m57t4.18d35b9f8b20afc2 namespace: kserve-ci-e2e-test resourceVersion: "16776" uid: 151542ba-4901-4b04-bbdb-15c5fff8a765 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-5d96855874-v5nt6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:18Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16779" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:18Z" message: Add eth0 [10.132.0.17/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-08T13:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-08T13:30:18Z" name: switch-graph-f794f-6766495c86-m57t4.18d35b9fad0a57c1 namespace: kserve-ci-e2e-test resourceVersion: "16793" uid: 97754254-e87b-4a20-b38e-e157d5c00d85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:18Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" metadata: creationTimestamp: "2026-09-08T13:30:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:18Z" name: switch-graph-f794f-6766495c86-m57t4.18d35b9faebd91c8 namespace: kserve-ci-e2e-test resourceVersion: "16795" uid: 70e6dd5d-9fbd-4d17-848e-5790905afd27 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:23Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:2ae619f4eea9fa406199e7b8e562bc31c295665fd082c5f377ac0602bcc5b2e9" in 5.396s (5.396s including waiting). Image size: 214921806 bytes.' metadata: creationTimestamp: "2026-09-08T13:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:23Z" name: switch-graph-f794f-6766495c86-m57t4.18d35ba0f06d5ac1 namespace: kserve-ci-e2e-test resourceVersion: "16836" uid: eb37689b-3623-43e1-ae87-e1f66733c398 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:23Z" message: 'Created container: switch-graph-f794f' metadata: creationTimestamp: "2026-09-08T13:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:23Z" name: switch-graph-f794f-6766495c86-m57t4.18d35ba0f325bad2 namespace: kserve-ci-e2e-test resourceVersion: "16838" uid: 3a643f43-8131-48fe-bb1c-9ce0ac88ba3b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:23Z" message: Started container switch-graph-f794f metadata: creationTimestamp: "2026-09-08T13:30:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:23Z" name: switch-graph-f794f-6766495c86-m57t4.18d35ba0f39997ac namespace: kserve-ci-e2e-test resourceVersion: "16839" uid: 59bfad75-a402-442e-894e-cdf6d774e99d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:30:31Z" message: Stopping container switch-graph-f794f metadata: creationTimestamp: "2026-09-08T13:30:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:30:31Z" name: switch-graph-f794f-6766495c86-m57t4.18d35ba2d6341a35 namespace: kserve-ci-e2e-test resourceVersion: "16926" uid: 8216d0d9-ba23-4748-8cf0-4676a344dd70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-08T13:30:35Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-f794f} kind: Pod name: switch-graph-f794f-6766495c86-m57t4 namespace: kserve-ci-e2e-test resourceVersion: "16771" uid: 3f84bd86-43cc-4f30-b38a-180837aba088 kind: Event lastTimestamp: "2026-09-08T13:31:00Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-08T13:30:35Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-08T13:31:00Z" name: switch-graph-f794f-6766495c86-m57t4.18d35ba38fc8700a namespace: kserve-ci-e2e-test resourceVersion: "17347" uid: c538c195-2e57-4266-9f95-f7f189f2d399 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-133-71.ec2.internal source: component: kubelet host: ip-10-0-133-71.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-f794f-6766495c86 namespace: kserve-ci-e2e-test resourceVersion: "16766" uid: 1ba168cc-8012-494e-a85f-beb4fbf5e16f kind: Event lastTimestamp: "2026-09-08T13:30:17Z" message: 'Created pod: switch-graph-f794f-6766495c86-m57t4' metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f-6766495c86.18d35b9f8a88155d namespace: kserve-ci-e2e-test resourceVersion: "16770" uid: 1aa4ad81-9669-4b86-b205-9afb2f271aaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-f794f namespace: kserve-ci-e2e-test resourceVersion: "16764" uid: ebe25750-e736-49db-9397-dd38c0920e2d kind: Event lastTimestamp: "2026-09-08T13:30:17Z" message: Scaled up replica set switch-graph-f794f-6766495c86 from 0 to 1 metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f.18d35b9f89a3b33b namespace: kserve-ci-e2e-test resourceVersion: "16767" uid: 9276545f-e303-4fd2-8d5f-ab74a3d6a4cf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f794f namespace: kserve-ci-e2e-test resourceVersion: "16781" uid: 4ca3085b-446b-4819-8c98-e6e7c7a22e08 kind: Event lastTimestamp: "2026-09-08T13:30:17Z" message: InferenceGraph [switch-graph-f794f] is Ready metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f.18d35b9f90f155a2 namespace: kserve-ci-e2e-test resourceVersion: "16784" uid: 26958984-f842-4cc2-b586-eceeedee92c6 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f794f namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 4ca3085b-446b-4819-8c98-e6e7c7a22e08 kind: Event lastTimestamp: "2026-09-08T13:30:17Z" message: 'Failed to update status for InferenceGraph "switch-graph-f794f": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-f794f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f.18d35b9f93d924a8 namespace: kserve-ci-e2e-test resourceVersion: "16787" uid: 339d683a-1f7a-494d-90c2-1cd37747b31b reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-08T13:30:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-f794f namespace: kserve-ci-e2e-test resourceVersion: "16759" uid: 4ca3085b-446b-4819-8c98-e6e7c7a22e08 kind: Event lastTimestamp: "2026-09-08T13:30:17Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "switch-graph-f794f": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-08T13:30:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-08T13:30:17Z" name: switch-graph-f794f.18d35b9f93d96cda namespace: kserve-ci-e2e-test resourceVersion: "16788" uid: e53292af-cdb1-464e-80aa-735b731e231c reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning kind: EventList metadata: resourceVersion: "35278"