--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:19:13.545657Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21152" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-1e5cb-df94dbf5b-cmf64 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:19:13Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62810911ef912 namespace: kserve-ci-e2e-test resourceVersion: "21157" uid: 18f4b530-1820-46a7-8e57-5083b9b5231c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:14Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21161" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:19:14Z" message: Add eth0 [10.132.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:19:14Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62810b32a9ab7 namespace: kserve-ci-e2e-test resourceVersion: "21177" uid: 33fa473e-dced-4242-8b2d-b4df96d76273 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1e5cb} kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:19:14Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:14Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62810b4ed65c7 namespace: kserve-ci-e2e-test resourceVersion: "21179" uid: 5c7d266e-fe99-47de-9e5a-09f88f1f71e9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1e5cb} kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:19:14Z" message: 'Created container: ensemble-graph-1e5cb' metadata: creationTimestamp: "2026-09-17T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:14Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62810b7ef7045 namespace: kserve-ci-e2e-test resourceVersion: "21180" uid: f2d8da67-aacc-45b3-b1ab-721449fe2f92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1e5cb} kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:19:14Z" message: Started container ensemble-graph-1e5cb metadata: creationTimestamp: "2026-09-17T16:19:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:14Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62810b8661006 namespace: kserve-ci-e2e-test resourceVersion: "21181" uid: 97908816-60d1-468a-b361-9ce470a41c70 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1e5cb} kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Stopping container ensemble-graph-1e5cb metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d62883be82625b namespace: kserve-ci-e2e-test resourceVersion: "24658" uid: bc99a34e-2e0a-4ce0-979a-6269b3c4b0d1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-1e5cb} kind: Pod name: ensemble-graph-1e5cb-df94dbf5b-cmf64 namespace: kserve-ci-e2e-test resourceVersion: "21153" uid: 113d312f-924a-45ab-9e61-4ca9d60086bf kind: Event lastTimestamp: "2026-09-17T16:27:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:56Z" name: ensemble-graph-1e5cb-df94dbf5b-cmf64.18d6288468355a91 namespace: kserve-ci-e2e-test resourceVersion: "24996" uid: 56fe14f8-0227-4dab-a1c1-767763d2a2ed reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-1e5cb-df94dbf5b namespace: kserve-ci-e2e-test resourceVersion: "21149" uid: 73cf9541-26b7-486a-a67e-26956ce2ede4 kind: Event lastTimestamp: "2026-09-17T16:19:13Z" message: 'Created pod: ensemble-graph-1e5cb-df94dbf5b-cmf64' metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:13Z" name: ensemble-graph-1e5cb-df94dbf5b.18d628109087c3a3 namespace: kserve-ci-e2e-test resourceVersion: "21154" uid: b5d93e08-31f2-4cb3-9df8-f248e722c467 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21147" uid: f95cfda3-4a95-44fe-94fa-0a8bd8e1961b kind: Event lastTimestamp: "2026-09-17T16:19:13Z" message: Scaled up replica set ensemble-graph-1e5cb-df94dbf5b from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:13Z" name: ensemble-graph-1e5cb.18d628108ebfa257 namespace: kserve-ci-e2e-test resourceVersion: "21150" uid: ffb51201-c6fb-45ae-9519-38dd1b8665a0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:13Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21163" uid: 594d73e6-fee4-4d35-8625-ca04b9018cae kind: Event lastTimestamp: "2026-09-17T16:19:13Z" message: InferenceGraph [ensemble-graph-1e5cb] is Ready metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:13Z" name: ensemble-graph-1e5cb.18d62810963d35c7 namespace: kserve-ci-e2e-test resourceVersion: "21165" uid: 2facbab1-098f-4a36-984d-7f3b74a97eb8 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: cef00dda-7675-4c60-ac3c-8fffd732009a kind: Event lastTimestamp: "2026-09-17T16:19:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:43Z" name: ensemble-graph-1e5cb.18d628140dac32f4 namespace: kserve-ci-e2e-test resourceVersion: "21432" uid: 96e8a87d-240c-45e5-8abb-725495ba0756 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21148" uid: cef00dda-7675-4c60-ac3c-8fffd732009a kind: Event lastTimestamp: "2026-09-17T16:19:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:43Z" name: ensemble-graph-1e5cb.18d628140e2c0299 namespace: kserve-ci-e2e-test resourceVersion: "21433" uid: 0f30803a-f79c-4051-89d0-1ac1e29bb581 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:03.073132Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-7c0d2-74876c585f-lhd4h to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:03Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d6280028a199c0 namespace: kserve-ci-e2e-test resourceVersion: "20193" uid: 019edf62-3f80-422a-9968-88a3e1f843f8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:03Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-7c0d2-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:03Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d62800331f6e57 namespace: kserve-ci-e2e-test resourceVersion: "20206" uid: 38fe33ff-2a8b-49b0-a1b6-6826b1becd0f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:04Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20198" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:04Z" message: Add eth0 [10.132.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:04Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d628006e6548f6 namespace: kserve-ci-e2e-test resourceVersion: "20217" uid: 74cb5cbc-c84d-4f08-a8e3-faf2b4c87d85 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7c0d2} kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:04Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:04Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d62800701c7307 namespace: kserve-ci-e2e-test resourceVersion: "20219" uid: a9c9c8df-068e-4c7c-bda4-ffe63bae1dd5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7c0d2} kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:04Z" message: 'Created container: ensemble-graph-7c0d2' metadata: creationTimestamp: "2026-09-17T16:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:04Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d6280072f9d57a namespace: kserve-ci-e2e-test resourceVersion: "20220" uid: 15198134-412a-4ba9-b77b-d95e7adf868a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7c0d2} kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:04Z" message: Started container ensemble-graph-7c0d2 metadata: creationTimestamp: "2026-09-17T16:18:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:04Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d62800736c6a81 namespace: kserve-ci-e2e-test resourceVersion: "20221" uid: 8de5b490-706f-4414-bde3-b99d4fad3df3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7c0d2} kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Stopping container ensemble-graph-7c0d2 metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d6280283db226f namespace: kserve-ci-e2e-test resourceVersion: "20310" uid: f4c29e37-27a4-4cf9-99e9-abc6c0e930ee reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-7c0d2} kind: Pod name: ensemble-graph-7c0d2-74876c585f-lhd4h namespace: kserve-ci-e2e-test resourceVersion: "20190" uid: 28709baf-99e9-4f16-a098-0b4ad19d9f18 kind: Event lastTimestamp: "2026-09-17T16:18:40Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:40Z" name: ensemble-graph-7c0d2-74876c585f-lhd4h.18d628031f856172 namespace: kserve-ci-e2e-test resourceVersion: "20679" uid: a8d569ae-5044-4a26-981e-741706b1536c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-7c0d2-74876c585f namespace: kserve-ci-e2e-test resourceVersion: "20181" uid: b843b03d-aacf-4ae9-a633-256145855ecb kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'Created pod: ensemble-graph-7c0d2-74876c585f-lhd4h' metadata: creationTimestamp: "2026-09-17T16:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: ensemble-graph-7c0d2-74876c585f.18d6280027aafa16 namespace: kserve-ci-e2e-test resourceVersion: "20189" uid: 20ec104d-39d5-4845-aba9-4396634233c3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "20180" uid: 9ff6b638-ce49-4cc8-b7ce-5f823ed899e4 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: Scaled up replica set ensemble-graph-7c0d2-74876c585f from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: ensemble-graph-7c0d2.18d6280025fa89db namespace: kserve-ci-e2e-test resourceVersion: "20185" uid: 57d249e0-f773-48a9-8edb-df26664f8f8d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:03Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "20200" uid: b2958f1f-dd06-4a08-ad42-012e7594e486 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: InferenceGraph [ensemble-graph-7c0d2] is Ready metadata: creationTimestamp: "2026-09-17T16:18:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:03Z" name: ensemble-graph-7c0d2.18d628002bf3a8c4 namespace: kserve-ci-e2e-test resourceVersion: "20202" uid: ce0c62a5-5e2e-4716-8e87-4350baa17aec reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:07:48.234405Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15136" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0dabd-predictor-7c8986597f-vl6mw to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d6277101632da5 namespace: kserve-ci-e2e-test resourceVersion: "15142" uid: 465af60e-c5a0-43cf-a63d-c16ae135b834 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-0dabd-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627710f165832 namespace: kserve-ci-e2e-test resourceVersion: "15154" uid: 8c440bfb-2a20-40e9-9107-15f33923b152 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15148" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d6277147148cc5 namespace: kserve-ci-e2e-test resourceVersion: "15180" uid: e8c584ef-cbd2-49cc-8c3b-f1d957fddb0b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d6277149907657 namespace: kserve-ci-e2e-test resourceVersion: "15183" uid: 8548529f-3afd-41f2-b6fa-79d5490a1321 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627714ca52212 namespace: kserve-ci-e2e-test resourceVersion: "15184" uid: c6e5d973-a029-468d-9e62-a455fe6714b8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627714d0900aa namespace: kserve-ci-e2e-test resourceVersion: "15185" uid: dd1745b2-a19b-4035-befb-e3e315319835 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627714d1477e9 namespace: kserve-ci-e2e-test resourceVersion: "15186" uid: d5ca2c41-3ea1-444d-8d75-293ce66f84b5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627714fc9dad7 namespace: kserve-ci-e2e-test resourceVersion: "15187" uid: d81a603a-5f93-4df6-a5a3-50d9d856aa48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:49Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:07:49Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:07:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:49Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d62771502718a2 namespace: kserve-ci-e2e-test resourceVersion: "15188" uid: 01cd30d8-d68d-4a1d-9df2-4396da77c47c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:08:26Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:26Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627718597d2c8 namespace: kserve-ci-e2e-test resourceVersion: "15534" uid: 36d3fb65-1cdb-485b-b5b9-3a602431869c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627f230a64dc4 namespace: kserve-ci-e2e-test resourceVersion: "19479" uid: 5d0fefa9-9b5c-4d2e-9ecb-8e5d89c1cb52 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw namespace: kserve-ci-e2e-test resourceVersion: "15140" uid: dd320469-feb2-4995-a991-be9db95c2b4a kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw.18d627f230a6c09f namespace: kserve-ci-e2e-test resourceVersion: "19483" uid: 26d5ab7d-fa58-4aac-acb3-32342fc4620c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0dabd-predictor-7c8986597f namespace: kserve-ci-e2e-test resourceVersion: "15129" uid: f33fb1d9-2d6b-40a6-a4ef-7994ac754898 kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: 'Created pod: error-404-isvc-0dabd-predictor-7c8986597f-vl6mw' metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-0dabd-predictor-7c8986597f.18d6277100dd5f13 namespace: kserve-ci-e2e-test resourceVersion: "15139" uid: a75dda8a-bae4-4d8d-a1f8-e00dd903eb4e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15126" uid: 612bba1a-f0ff-4c13-a239-ae840f9a097c kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Scaled up replica set error-404-isvc-0dabd-predictor-7c8986597f from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-0dabd-predictor.18d62770ff0873e1 namespace: kserve-ci-e2e-test resourceVersion: "15130" uid: eec289c7-c5fe-46e3-adb8-c975607c365f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 0a939f37-f05b-41f7-84fe-0041473d93a6 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:48Z" name: error-404-isvc-0dabd-predictor.18d627747ddc8ce0 namespace: kserve-ci-e2e-test resourceVersion: "15921" uid: 7a4554e7-3738-4ba2-96c0-624afd68f187 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15128" uid: 0a939f37-f05b-41f7-84fe-0041473d93a6 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:48Z" name: error-404-isvc-0dabd-predictor.18d627747e5df7b3 namespace: kserve-ci-e2e-test resourceVersion: "15923" uid: 7af772f3-0fc7-42e7-bfc9-3d58028152c1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15112" uid: da76f446-93c1-417d-9be9-b8216fb46252 kind: Event lastTimestamp: "2026-09-17T16:08:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:36Z" name: error-404-isvc-0dabd.18d62770ff02a0e0 namespace: kserve-ci-e2e-test resourceVersion: "15800" uid: 2b0a35d4-b949-4cea-ba74-9334a2c91de0 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15795" uid: da76f446-93c1-417d-9be9-b8216fb46252 kind: Event lastTimestamp: "2026-09-17T16:08:36Z" message: InferenceService [error-404-isvc-0dabd] is Ready metadata: creationTimestamp: "2026-09-17T16:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:36Z" name: error-404-isvc-0dabd.18d6277c4384680f namespace: kserve-ci-e2e-test resourceVersion: "15797" uid: 2a6e48d8-aee0-4a86-977d-80421c85cedd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:13.777526Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20398" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802a6a9b73b namespace: kserve-ci-e2e-test resourceVersion: "20402" uid: 1ee995ea-6f4c-4b18-8471-ca353595d1fa reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-1e5cb-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802aeedafea namespace: kserve-ci-e2e-test resourceVersion: "20414" uid: 34d35342-fa74-46fa-87b0-488c6cb105c6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20409" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:14Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802ecb16e7e namespace: kserve-ci-e2e-test resourceVersion: "20438" uid: a1f5445e-2326-4a31-881b-dfc4439c0aa2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802eeb81f47 namespace: kserve-ci-e2e-test resourceVersion: "20440" uid: 96947ef8-9d3e-48d1-ad31-1417055039e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802f20ad6da namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: df578dd9-d66c-44eb-9be8-a455cddef5a5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802f27abe37 namespace: kserve-ci-e2e-test resourceVersion: "20443" uid: 004ad80e-c905-45e1-ac97-2d333bea0c2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802f28489d8 namespace: kserve-ci-e2e-test resourceVersion: "20444" uid: a69ea517-28a4-45b5-b69f-48b0f9e59d26 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802f511311a namespace: kserve-ci-e2e-test resourceVersion: "20445" uid: cdb25d85-0cb3-4dda-a23e-378a4c5e3711 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802f58373d0 namespace: kserve-ci-e2e-test resourceVersion: "20446" uid: be1f0834-eeff-43e9-b059-2434e8bf8709 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:27:31Z" message: 'Readiness probe failed: dial tcp 10.134.0.35:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:31Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62802fd94b8f4 namespace: kserve-ci-e2e-test resourceVersion: "24799" uid: d433140d-1c54-4ed9-b85f-2d6b54248499 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62883dbbca87c namespace: kserve-ci-e2e-test resourceVersion: "24719" uid: d14720a3-c805-40a3-8d88-46c6e50b4a09 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d62883dbbd34d2 namespace: kserve-ci-e2e-test resourceVersion: "24720" uid: 73f4a641-f634-4137-8236-684abf495bc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp namespace: kserve-ci-e2e-test resourceVersion: "20400" uid: f82dd29d-21f3-459b-a036-700eca9352f2 kind: Event lastTimestamp: "2026-09-17T16:27:31Z" message: 'Readiness probe failed: Get "https://10.134.0.35:8643/healthz": dial tcp 10.134.0.35:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:31Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp.18d6288471f7258a namespace: kserve-ci-e2e-test resourceVersion: "24798" uid: 323a1dfd-e246-4b76-9fcd-dfd49b80daf4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-1e5cb-predictor-79cdb767cc namespace: kserve-ci-e2e-test resourceVersion: "20389" uid: 7291a152-4630-40f8-a8ec-194a69b52f64 kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: 'Created pod: error-404-isvc-1e5cb-predictor-79cdb767cc-t5lsp' metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-1e5cb-predictor-79cdb767cc.18d62802a5fc01a9 namespace: kserve-ci-e2e-test resourceVersion: "20399" uid: cd67955e-136f-4fc5-8d02-6d344f640dae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20386" uid: 35d97a11-8345-4196-b075-57b17c4c2d0e kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Scaled up replica set error-404-isvc-1e5cb-predictor-79cdb767cc from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-1e5cb-predictor.18d62802a1b6dd3d namespace: kserve-ci-e2e-test resourceVersion: "20392" uid: 2ecc558a-e90b-47db-9f91-b8a9a184d003 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 4370095a-b193-4cbf-9f2d-2d1f3a01fd56 kind: Event lastTimestamp: "2026-09-17T16:18:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:58Z" name: error-404-isvc-1e5cb-predictor.18d628061fef7ea7 namespace: kserve-ci-e2e-test resourceVersion: "21002" uid: 9e47cbce-c846-4c65-b8fb-fd232fc2f24d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20387" uid: 4370095a-b193-4cbf-9f2d-2d1f3a01fd56 kind: Event lastTimestamp: "2026-09-17T16:18:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:58Z" name: error-404-isvc-1e5cb-predictor.18d628062093a33d namespace: kserve-ci-e2e-test resourceVersion: "21003" uid: eb8f9395-31e9-42f7-ac08-a1d21e3d4e8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 4370095a-b193-4cbf-9f2d-2d1f3a01fd56 kind: Event lastTimestamp: "2026-09-17T16:19:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:28Z" name: error-404-isvc-1e5cb-predictor.18d62810a0c8f680 namespace: kserve-ci-e2e-test resourceVersion: "21305" uid: c31fcece-5516-4821-8f08-1410177a9f9d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20568" uid: 4370095a-b193-4cbf-9f2d-2d1f3a01fd56 kind: Event lastTimestamp: "2026-09-17T16:19:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:28Z" name: error-404-isvc-1e5cb-predictor.18d62810a17952f4 namespace: kserve-ci-e2e-test resourceVersion: "21306" uid: 71443e68-20a7-4cd3-86c3-2f559855df11 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "20371" uid: 08c0b0a2-e190-4997-ae4b-33041a40991a kind: Event lastTimestamp: "2026-09-17T16:19:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:01Z" name: error-404-isvc-1e5cb.18d62802a1a527b7 namespace: kserve-ci-e2e-test resourceVersion: "21033" uid: ffc7f7cf-cb35-4c10-a1fd-089346b59a34 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21029" uid: 08c0b0a2-e190-4997-ae4b-33041a40991a kind: Event lastTimestamp: "2026-09-17T16:19:01Z" message: InferenceService [error-404-isvc-1e5cb] is Ready metadata: creationTimestamp: "2026-09-17T16:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:01Z" name: error-404-isvc-1e5cb.18d6280dbb97addd namespace: kserve-ci-e2e-test resourceVersion: "21030" uid: b1710378-fbb1-448d-99dd-83fb9fb24402 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:54.193619Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20918" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-26f37-predictor-64fcbb98df-qdjgh to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c0fa665cf namespace: kserve-ci-e2e-test resourceVersion: "20922" uid: d21eb579-c084-45df-ae64-6e68c5479259 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-26f37-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c1f7b3a63 namespace: kserve-ci-e2e-test resourceVersion: "20935" uid: aeededfd-23ca-4965-a0dd-3f1bef99697f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20928" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c55747f6e namespace: kserve-ci-e2e-test resourceVersion: "20955" uid: 494c0015-3f64-403a-8bea-e9e3e1045996 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c58101cb2 namespace: kserve-ci-e2e-test resourceVersion: "20957" uid: a08598e1-6e01-4b4a-ad8d-4d669bc0b44b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c5c26d70f namespace: kserve-ci-e2e-test resourceVersion: "20959" uid: 180f9bb0-5235-4f56-897b-8a9f4ecc7836 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c5c9fd0b6 namespace: kserve-ci-e2e-test resourceVersion: "20960" uid: 5db95f6c-a19e-456f-a8a9-c1dc050c297d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c5ca9d0f7 namespace: kserve-ci-e2e-test resourceVersion: "20961" uid: e350334b-638c-4c65-9b3b-8d93a0652dda reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c60330909 namespace: kserve-ci-e2e-test resourceVersion: "20962" uid: 7b002df7-04a3-4220-a394-b1dc150f45a2 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:18:55Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:55Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c60ba6d09 namespace: kserve-ci-e2e-test resourceVersion: "20963" uid: df1b64a7-c9e6-4d88-94f4-22815cdbd338 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:19:32Z" message: 'Readiness probe failed: dial tcp 10.134.0.36:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:32Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6280c94febe4c namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 220f5196-03e1-46c8-b931-db8f0df0c8a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6288d2b74055a namespace: kserve-ci-e2e-test resourceVersion: "25155" uid: 7a69cd42-6167-4e8d-84ae-8f9f72cad357 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh namespace: kserve-ci-e2e-test resourceVersion: "20919" uid: 30952538-669f-4d74-960f-ae136009a7bc kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh.18d6288d2b74819f namespace: kserve-ci-e2e-test resourceVersion: "25157" uid: ed360cee-459c-4490-bcc6-dbff49c26d74 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-26f37-predictor-64fcbb98df namespace: kserve-ci-e2e-test resourceVersion: "20911" uid: e5e1605a-cce9-44fd-b09d-e67fb91614d9 kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'Created pod: error-404-isvc-26f37-predictor-64fcbb98df-qdjgh' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37-predictor-64fcbb98df.18d6280c0f14505e namespace: kserve-ci-e2e-test resourceVersion: "20920" uid: c5708540-dcb1-426a-b4e9-d75592118878 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20908" uid: f2145f06-4288-4146-b353-2348a9d1ff59 kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Scaled up replica set error-404-isvc-26f37-predictor-64fcbb98df from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37-predictor.18d6280c09f8dc8d namespace: kserve-ci-e2e-test resourceVersion: "20912" uid: f1d305a2-f811-49fe-80ce-2a95c680443e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: d14ebb26-5ad2-45cd-928d-8a3457a26c2f kind: Event lastTimestamp: "2026-09-17T16:19:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:24Z" name: error-404-isvc-26f37-predictor.18d6280f889d3849 namespace: kserve-ci-e2e-test resourceVersion: "21265" uid: ffc578f0-1274-4385-8fe1-89ea86dce13a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:19:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20910" uid: d14ebb26-5ad2-45cd-928d-8a3457a26c2f kind: Event lastTimestamp: "2026-09-17T16:19:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:24Z" name: error-404-isvc-26f37-predictor.18d6280f8915e8df namespace: kserve-ci-e2e-test resourceVersion: "21266" uid: 6a02e85a-037b-4c42-81cf-d9e89571eb89 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:19:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: d14ebb26-5ad2-45cd-928d-8a3457a26c2f kind: Event lastTimestamp: "2026-09-17T16:20:24Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:24Z" name: error-404-isvc-26f37-predictor.18d62816878ef5f2 namespace: kserve-ci-e2e-test resourceVersion: "21770" uid: 3b019f9d-be06-42cd-a68b-4cc68051c3bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:19:39Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "21106" uid: d14ebb26-5ad2-45cd-928d-8a3457a26c2f kind: Event lastTimestamp: "2026-09-17T16:20:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:24Z" name: error-404-isvc-26f37-predictor.18d6281688139c2f namespace: kserve-ci-e2e-test resourceVersion: "21771" uid: 6a3b9958-5b6d-41e4-89a2-6407afb26035 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: daa908ef-5849-4064-83d0-89648819249d kind: Event lastTimestamp: "2026-09-17T16:19:42Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:42Z" name: error-404-isvc-26f37.18d6280c0a4849c3 namespace: kserve-ci-e2e-test resourceVersion: "21424" uid: 2e868323-b6a1-41c1-8484-4a4b01ece959 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: daa908ef-5849-4064-83d0-89648819249d kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'Failed to update status for InferenceService "error-404-isvc-26f37": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37.18d6280c0bb57f38 namespace: kserve-ci-e2e-test resourceVersion: "20915" uid: eeb80572-573b-494a-93a7-1e4eec777ed2 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20889" uid: daa908ef-5849-4064-83d0-89648819249d kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-26f37.18d6280c0bb5c8ec namespace: kserve-ci-e2e-test resourceVersion: "20917" uid: ed56815e-63a1-4e1f-b24e-4c6d8f30eb39 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:42Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21421" uid: daa908ef-5849-4064-83d0-89648819249d kind: Event lastTimestamp: "2026-09-17T16:19:42Z" message: InferenceService [error-404-isvc-26f37] is Ready metadata: creationTimestamp: "2026-09-17T16:19:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:42Z" name: error-404-isvc-26f37.18d6281753d93f27 namespace: kserve-ci-e2e-test resourceVersion: "21422" uid: edf80dc0-bc93-4627-8cc8-246a404d1d2e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:28:08.932370Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25191" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-37c70-predictor-5655c4d7c-tlmsr to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d38a7e005 namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: 4c375cd2-6e16-42d4-8872-00b2388429fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25202" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d5ab6e6b1 namespace: kserve-ci-e2e-test resourceVersion: "25223" uid: aca0f8ad-2195-4df9-bb75-da8e6515df30 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d5c30ab43 namespace: kserve-ci-e2e-test resourceVersion: "25225" uid: 948cc6a9-9ea2-4070-9e06-5304089c5c45 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d5ef1067d namespace: kserve-ci-e2e-test resourceVersion: "25226" uid: 0862f00f-15db-4fa9-8e0f-9dff44f89060 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d5f6240cb namespace: kserve-ci-e2e-test resourceVersion: "25227" uid: 6c7bb911-1536-4b81-a916-45c14bf1d338 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d5f6c9cb0 namespace: kserve-ci-e2e-test resourceVersion: "25228" uid: cfd63a3e-b87a-4381-89f2-8d6068d816ad reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d62b20e1a namespace: kserve-ci-e2e-test resourceVersion: "25229" uid: 87211e4b-2277-456d-985e-3cfcf1d17333 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288d631f13a7 namespace: kserve-ci-e2e-test resourceVersion: "25230" uid: cfe23d35-9154-4432-8c89-27b44f557ece reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:28:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:46Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6288dae85aa87 namespace: kserve-ci-e2e-test resourceVersion: "25748" uid: 77538e2e-42a5-4a44-9856-81a6519b86ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:45:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:28Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6297f40b5d158 namespace: kserve-ci-e2e-test resourceVersion: "33146" uid: af5db233-44f8-4486-95a9-5e66c2e62d48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr namespace: kserve-ci-e2e-test resourceVersion: "25193" uid: de8efe43-779c-413c-9a5b-db90a22c4a15 kind: Event lastTimestamp: "2026-09-17T16:45:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:28Z" name: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr.18d6297f40b69401 namespace: kserve-ci-e2e-test resourceVersion: "33147" uid: 3c19844c-0ff5-4538-81a2-208cec5f71ed reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-37c70-predictor-5655c4d7c namespace: kserve-ci-e2e-test resourceVersion: "25182" uid: 3143bb59-c040-4ba7-b72a-366b143d96e0 kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'Created pod: error-404-isvc-37c70-predictor-5655c4d7c-tlmsr' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-37c70-predictor-5655c4d7c.18d6288d37d843a5 namespace: kserve-ci-e2e-test resourceVersion: "25192" uid: 78c3814e-042a-4a0d-8bd7-08795843bf64 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25180" uid: c3525612-f078-4133-9123-f04a3265d7be kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Scaled up replica set error-404-isvc-37c70-predictor-5655c4d7c from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-37c70-predictor.18d6288d3578ecb2 namespace: kserve-ci-e2e-test resourceVersion: "25184" uid: cbd5d4fc-2ea9-4b6a-b860-2320b54fd96b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 00384275-4c57-4886-adb0-13a5e5fe7c31 kind: Event lastTimestamp: "2026-09-17T16:28:23Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:23Z" name: error-404-isvc-37c70-predictor.18d62890b4ac853a namespace: kserve-ci-e2e-test resourceVersion: "25378" uid: 30dee5af-951b-414f-b31b-d49ef981a790 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25181" uid: 00384275-4c57-4886-adb0-13a5e5fe7c31 kind: Event lastTimestamp: "2026-09-17T16:28:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:23Z" name: error-404-isvc-37c70-predictor.18d62890b5678013 namespace: kserve-ci-e2e-test resourceVersion: "25380" uid: 98d6ef3d-1205-4198-95f8-e5f29c5d58a8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:28:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 00384275-4c57-4886-adb0-13a5e5fe7c31 kind: Event lastTimestamp: "2026-09-17T16:29:08Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:08Z" name: error-404-isvc-37c70-predictor.18d62894348cef7c namespace: kserve-ci-e2e-test resourceVersion: "25972" uid: 78c7371c-47fd-43db-8b97-1b8bc80a1790 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:28:38Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25379" uid: 00384275-4c57-4886-adb0-13a5e5fe7c31 kind: Event lastTimestamp: "2026-09-17T16:29:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:09Z" name: error-404-isvc-37c70-predictor.18d62894355ae166 namespace: kserve-ci-e2e-test resourceVersion: "25974" uid: 39ae7ea9-1469-4d74-8e28-71c57087a2fc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: feaf42ce-e9b3-44dd-bc51-de3a5ddbd5ba kind: Event lastTimestamp: "2026-09-17T16:28:57Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:57Z" name: error-404-isvc-37c70.18d6288d357c512f namespace: kserve-ci-e2e-test resourceVersion: "25854" uid: 2285fffd-8555-4170-8358-2ea9b80e1265 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: feaf42ce-e9b3-44dd-bc51-de3a5ddbd5ba kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'Failed to update status for InferenceService "error-404-isvc-37c70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-37c70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-37c70.18d6288d362f8239 namespace: kserve-ci-e2e-test resourceVersion: "25186" uid: e2584129-1631-4e4b-a501-61a2f05c15b6 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25172" uid: feaf42ce-e9b3-44dd-bc51-de3a5ddbd5ba kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-37c70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:08Z" name: error-404-isvc-37c70.18d6288d362fe09e namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: ed6af791-994e-4473-98b0-5a84e00b5a91 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25851" uid: feaf42ce-e9b3-44dd-bc51-de3a5ddbd5ba kind: Event lastTimestamp: "2026-09-17T16:28:57Z" message: InferenceService [error-404-isvc-37c70] is Ready metadata: creationTimestamp: "2026-09-17T16:28:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:57Z" name: error-404-isvc-37c70.18d628986e0da9b9 namespace: kserve-ci-e2e-test resourceVersion: "25853" uid: d9638f5e-d74d-4eaa-ba8b-a2d5f2894e7a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:17:43.736281Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19948" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-48574-predictor-768499d5df-spw9k to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fba810aeae namespace: kserve-ci-e2e-test resourceVersion: "19952" uid: 7ec97f68-a15a-466d-aae7-b72ef082755d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-48574-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbb6b709ac namespace: kserve-ci-e2e-test resourceVersion: "19965" uid: 2036b4a9-53a0-414f-b838-53161e3c39c0 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19958" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:17:44Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbee37bd34 namespace: kserve-ci-e2e-test resourceVersion: "19984" uid: a2178f7e-6766-471a-8182-293f51d30d06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbefb9f6e0 namespace: kserve-ci-e2e-test resourceVersion: "19986" uid: 3e6c4e69-41c0-43ad-8144-d3140cca468b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf2a90252 namespace: kserve-ci-e2e-test resourceVersion: "19989" uid: 005e4dbd-fecb-48ec-aa1c-9abf81750b94 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:45Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf3507f23 namespace: kserve-ci-e2e-test resourceVersion: "19991" uid: a5171160-a346-4fee-9b43-7fa76a8d7c3e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:45Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf35ad397 namespace: kserve-ci-e2e-test resourceVersion: "19992" uid: 79e533e4-2184-4c49-be0f-32c977f9f23e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:45Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:45Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf61ab246 namespace: kserve-ci-e2e-test resourceVersion: "19994" uid: 73baa52b-7b44-41fe-9491-84e694909204 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:17:45Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:45Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf69bb214 namespace: kserve-ci-e2e-test resourceVersion: "19995" uid: 9322bb4e-32f9-42a3-8983-7460e0df99f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:18:21Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:21Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d627fbf8c80298 namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 42204d51-2b56-4166-9843-a39b1f6ec7bf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d6280c0400078d namespace: kserve-ci-e2e-test resourceVersion: "20896" uid: 423854ef-e0a4-49ba-85f2-c53353f43201 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d6280c04009734 namespace: kserve-ci-e2e-test resourceVersion: "20898" uid: 18faaaa1-997b-4345-9dc4-7240ce75df38 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48574-predictor-768499d5df-spw9k namespace: kserve-ci-e2e-test resourceVersion: "19950" uid: fc0b2309-ae47-4647-b7c9-dfc97cdc6aae kind: Event lastTimestamp: "2026-09-17T16:18:56Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:56Z" name: error-404-isvc-48574-predictor-768499d5df-spw9k.18d6280c8101a513 namespace: kserve-ci-e2e-test resourceVersion: "20969" uid: 11ec8bf3-7b79-4e77-a00a-165dbdec0a40 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-48574-predictor-768499d5df namespace: kserve-ci-e2e-test resourceVersion: "19942" uid: 2e9bc0ac-2ae8-40a5-9deb-8804949431e8 kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: 'Created pod: error-404-isvc-48574-predictor-768499d5df-spw9k' metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-48574-predictor-768499d5df.18d627fba798e44c namespace: kserve-ci-e2e-test resourceVersion: "19949" uid: c36bc46d-f7ff-4586-8b4b-5f5d61b0561b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19940" uid: 8418d7cc-def4-4ff6-94b5-6d929e54edbe kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Scaled up replica set error-404-isvc-48574-predictor-768499d5df from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-48574-predictor.18d627fba53e98e8 namespace: kserve-ci-e2e-test resourceVersion: "19943" uid: 75a400cc-fcd3-4070-8181-e1d08ea6adb3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:17:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: acf70b90-5fe9-493f-a677-fb12629f6ec1 kind: Event lastTimestamp: "2026-09-17T16:18:28Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:28Z" name: error-404-isvc-48574-predictor.18d627ff23fb99db namespace: kserve-ci-e2e-test resourceVersion: "20571" uid: 658b013a-2280-4d9e-8654-63f0807ca14d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:17:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19941" uid: acf70b90-5fe9-493f-a677-fb12629f6ec1 kind: Event lastTimestamp: "2026-09-17T16:18:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:28Z" name: error-404-isvc-48574-predictor.18d627ff246c2c68 namespace: kserve-ci-e2e-test resourceVersion: "20572" uid: 91356fa6-0f23-4007-911d-6cf2d616257a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: acf70b90-5fe9-493f-a677-fb12629f6ec1 kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: error-404-isvc-48574-predictor.18d62802a3aad13a namespace: kserve-ci-e2e-test resourceVersion: "20729" uid: 6202b05d-7bf4-40c5-9fd6-66ac3f210ee9 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "20137" uid: acf70b90-5fe9-493f-a677-fb12629f6ec1 kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: error-404-isvc-48574-predictor.18d62802a4912bc6 namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 87648cf0-ff40-4b64-aaf3-921bb1de9ce9 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48574 namespace: kserve-ci-e2e-test resourceVersion: "19929" uid: d1f33213-3836-48bc-8dd1-9eb67358ba3e kind: Event lastTimestamp: "2026-09-17T16:18:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:31Z" name: error-404-isvc-48574.18d627fba544cecd namespace: kserve-ci-e2e-test resourceVersion: "20599" uid: db3fe3e1-c424-4452-a547-e011001037ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48574 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: d1f33213-3836-48bc-8dd1-9eb67358ba3e kind: Event lastTimestamp: "2026-09-17T16:18:31Z" message: InferenceService [error-404-isvc-48574] is Ready metadata: creationTimestamp: "2026-09-17T16:18:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:31Z" name: error-404-isvc-48574.18d62806baed8552 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: 3fcdcfe3-b40a-452e-b5d9-32c2e48bbab8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:05:58.381599Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13948" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-48b6f-predictor-98c8c6df-zv5fb to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:05:58Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d627576da6654e namespace: kserve-ci-e2e-test resourceVersion: "13952" uid: 780b25dc-c370-4d02-9c94-b0102742b0bb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-48b6f-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d627577721ee24 namespace: kserve-ci-e2e-test resourceVersion: "13963" uid: 4237bb80-c970-40e5-85ce-0e97d6d1105f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13955" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:05:59Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d62757b37f9717 namespace: kserve-ci-e2e-test resourceVersion: "14021" uid: fe182e2d-bd14-4266-84c4-d2ee7a574138 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:59Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d62757b5f7420f namespace: kserve-ci-e2e-test resourceVersion: "14026" uid: 21689b99-4d5e-42c0-8276-4f5c4da30bc8 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:11Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" in 11.785s (11.785s including waiting). Image size: 1357371469 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:11Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275a746c9683 namespace: kserve-ci-e2e-test resourceVersion: "14147" uid: a6214fda-6fe8-4617-b2c0-a6b23c946a19 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:11Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:11Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275a77cd177e namespace: kserve-ci-e2e-test resourceVersion: "14149" uid: 138460b5-95ef-4064-8990-c637945f5bf0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:11Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:11Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275a782c4f79 namespace: kserve-ci-e2e-test resourceVersion: "14150" uid: fb7bf1f2-3dfc-4c1a-b110-b755e2dca815 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:11Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:11Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T16:06:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:11Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275a78392b54 namespace: kserve-ci-e2e-test resourceVersion: "14151" uid: feb62be9-e064-4bc7-b12d-bb267a01206a reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:13Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.522s (2.522s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:13Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275b0e8ddeef namespace: kserve-ci-e2e-test resourceVersion: "14180" uid: 97225f71-3318-47d7-abd5-62111991dc68 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:14Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275b11b23d52 namespace: kserve-ci-e2e-test resourceVersion: "14181" uid: 66766eda-ff41-40db-b338-0feee9f81477 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:06:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:14Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275b12763866 namespace: kserve-ci-e2e-test resourceVersion: "14182" uid: 37d71ce5-867d-44e7-aef2-898579af4e21 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T16:06:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:06:52Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:06:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:52Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d6275b4e84764f namespace: kserve-ci-e2e-test resourceVersion: "14523" uid: c5533cd3-fba2-439b-b745-49d57eb33550 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d62770f3867fb9 namespace: kserve-ci-e2e-test resourceVersion: "15096" uid: 71254b42-013f-4efd-827d-fbc73da1d9e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb namespace: kserve-ci-e2e-test resourceVersion: "13949" uid: fc0a8ab5-777e-492a-be25-6689499eb0e5 kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb.18d62770f38701de namespace: kserve-ci-e2e-test resourceVersion: "15097" uid: bebfeee3-56c3-4e63-90f8-10b0134aa944 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-48b6f-predictor-98c8c6df namespace: kserve-ci-e2e-test resourceVersion: "13942" uid: 9a525ecc-6890-45c7-b71d-a129941e54b0 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'Created pod: error-404-isvc-48b6f-predictor-98c8c6df-zv5fb' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: error-404-isvc-48b6f-predictor-98c8c6df.18d627576d30b632 namespace: kserve-ci-e2e-test resourceVersion: "13950" uid: 30a0456b-6d73-45e7-9cad-fbb92fb47fb5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13939" uid: f627b32d-15b8-4e79-b0e0-cc1358d12d5a kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Scaled up replica set error-404-isvc-48b6f-predictor-98c8c6df from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: error-404-isvc-48b6f-predictor.18d627576a79c554 namespace: kserve-ci-e2e-test resourceVersion: "13945" uid: 7999b661-3e20-49d0-a508-7124c87db49a reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: 7fd0325d-e108-4119-b5d4-c2db840996fa kind: Event lastTimestamp: "2026-09-17T16:06:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:43Z" name: error-404-isvc-48b6f-predictor.18d6275ae8c21152 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: 05d00541-5b13-40fe-b0a1-a7c0f2720569 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13940" uid: 7fd0325d-e108-4119-b5d4-c2db840996fa kind: Event lastTimestamp: "2026-09-17T16:06:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:43Z" name: error-404-isvc-48b6f-predictor.18d6275ae934d02e namespace: kserve-ci-e2e-test resourceVersion: "14461" uid: fd16f3e9-ab86-493a-9e22-e7c4e2d2038d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: 7fd0325d-e108-4119-b5d4-c2db840996fa kind: Event lastTimestamp: "2026-09-17T16:07:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:28Z" name: error-404-isvc-48b6f-predictor.18d627656676307f namespace: kserve-ci-e2e-test resourceVersion: "14837" uid: 073c05c7-d58c-42f0-9a3d-8e935f51e3dd reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14171" uid: 7fd0325d-e108-4119-b5d4-c2db840996fa kind: Event lastTimestamp: "2026-09-17T16:07:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:28Z" name: error-404-isvc-48b6f-predictor.18d6276566ff999c namespace: kserve-ci-e2e-test resourceVersion: "14838" uid: 7673721a-879a-4cff-bdf8-f646fac1b3f0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48b6f namespace: kserve-ci-e2e-test resourceVersion: "13922" uid: ded738bb-8621-42c7-a8d2-5e1b547fd108 kind: Event lastTimestamp: "2026-09-17T16:07:02Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:02Z" name: error-404-isvc-48b6f.18d6275769be661b namespace: kserve-ci-e2e-test resourceVersion: "14621" uid: a1a9c9cb-0c6e-4361-9fd7-d7fcbdf3c149 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-48b6f namespace: kserve-ci-e2e-test resourceVersion: "14618" uid: ded738bb-8621-42c7-a8d2-5e1b547fd108 kind: Event lastTimestamp: "2026-09-17T16:07:02Z" message: InferenceService [error-404-isvc-48b6f] is Ready metadata: creationTimestamp: "2026-09-17T16:07:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:02Z" name: error-404-isvc-48b6f.18d627664a7f0fb6 namespace: kserve-ci-e2e-test resourceVersion: "14619" uid: e11f9445-2ee7-4d2d-a1f8-96afedb43208 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:08:29.238879Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15674" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-4f67a-predictor-5476476989-b6s8s to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277a8d71db7f namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 4ced41a7-1077-4c74-a2ee-ffaf2ef44a38 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15682" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277aaf8a883d namespace: kserve-ci-e2e-test resourceVersion: "15694" uid: 91325687-a3c3-4f31-9ebb-a3373be672f8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab1a2a1d3 namespace: kserve-ci-e2e-test resourceVersion: "15697" uid: 11785539-cb22-4352-b305-a10539a1f51f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab435b285 namespace: kserve-ci-e2e-test resourceVersion: "15698" uid: d9f6811b-eb77-43f4-9b87-9c4fe3f32c48 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab49b566e namespace: kserve-ci-e2e-test resourceVersion: "15699" uid: 6939148c-b16b-4c6f-9de8-ce3cef41ddd8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab4a510e8 namespace: kserve-ci-e2e-test resourceVersion: "15700" uid: 621799cb-9051-4f39-9e00-a0088f613e5d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab7755bdf namespace: kserve-ci-e2e-test resourceVersion: "15701" uid: 1ae33bb2-645b-46e5-892f-4f038ad9f797 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ab7d89c11 namespace: kserve-ci-e2e-test resourceVersion: "15702" uid: b97add1b-0fe6-41a7-a097-3e16f9fb6557 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-09-17T16:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:17:46Z" message: 'Readiness probe failed: dial tcp 10.134.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:46Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d6277ae032d2f6 namespace: kserve-ci-e2e-test resourceVersion: "20018" uid: 1040a262-3661-4980-b356-a7cc3afadc7e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d627fb9acf0855 namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: 387b2428-0847-4306-9046-bbe101cecd82 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d627fb9acf90ab namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: 8e7efc91-3642-4016-a632-d9c5d9097243 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:46Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-4f67a-predictor-5476476989-b6s8s namespace: kserve-ci-e2e-test resourceVersion: "15675" uid: e2182e6a-6b5b-49aa-a591-1f55b4abdd8c kind: Event lastTimestamp: "2026-09-17T16:17:46Z" message: 'Readiness probe failed: Get "https://10.134.0.32:8643/healthz": dial tcp 10.134.0.32:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:17:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:46Z" name: error-404-isvc-4f67a-predictor-5476476989-b6s8s.18d627fc549f8489 namespace: kserve-ci-e2e-test resourceVersion: "20017" uid: b00bd402-a639-4125-ae7f-29c99d4ae0a0 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-4f67a-predictor-5476476989 namespace: kserve-ci-e2e-test resourceVersion: "15665" uid: 0dfb0291-9e24-4cd2-ac23-274950cf0e14 kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: 'Created pod: error-404-isvc-4f67a-predictor-5476476989-b6s8s' metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor-5476476989.18d6277a8cadef14 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 85a1ea13-2608-42a3-b2b0-cbde45476e26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15663" uid: cf344b44-5c65-46a9-a4d3-68139d7b914c kind: Event lastTimestamp: "2026-09-17T16:08:29Z" message: Scaled up replica set error-404-isvc-4f67a-predictor-5476476989 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:29Z" name: error-404-isvc-4f67a-predictor.18d6277a870a60a4 namespace: kserve-ci-e2e-test resourceVersion: "15668" uid: f6a36c5b-2510-4d10-b6a0-3e1437afb19f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 543b4bdb-f9ee-46e5-bf45-8f8cbe8ee201 kind: Event lastTimestamp: "2026-09-17T16:08:59Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:59Z" name: error-404-isvc-4f67a-predictor.18d6277e07dbb5b9 namespace: kserve-ci-e2e-test resourceVersion: "16024" uid: 40df8746-0b67-48af-b8f8-aea9592e6b51 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:44Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15666" uid: 543b4bdb-f9ee-46e5-bf45-8f8cbe8ee201 kind: Event lastTimestamp: "2026-09-17T16:08:59Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:59Z" name: error-404-isvc-4f67a-predictor.18d6277e086de43f namespace: kserve-ci-e2e-test resourceVersion: "16025" uid: b0f62326-45b2-48f1-a034-56e75cf8e1b4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:09:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 543b4bdb-f9ee-46e5-bf45-8f8cbe8ee201 kind: Event lastTimestamp: "2026-09-17T16:09:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:44Z" name: error-404-isvc-4f67a-predictor.18d6278507d3d9d7 namespace: kserve-ci-e2e-test resourceVersion: "16415" uid: 5f53808f-5416-4c81-b8cf-837ec70b725c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:09:14Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15863" uid: 543b4bdb-f9ee-46e5-bf45-8f8cbe8ee201 kind: Event lastTimestamp: "2026-09-17T16:09:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:09:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:44Z" name: error-404-isvc-4f67a-predictor.18d62785087ca34a namespace: kserve-ci-e2e-test resourceVersion: "16416" uid: 7a85521f-709b-44b9-991e-82725c60c74c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:08:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f67a namespace: kserve-ci-e2e-test resourceVersion: "15651" uid: bb50e861-859f-4c0c-a0fc-97cc69a9cfc1 kind: Event lastTimestamp: "2026-09-17T16:09:16Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:09:16Z" name: error-404-isvc-4f67a.18d6277a88c05e70 namespace: kserve-ci-e2e-test resourceVersion: "16167" uid: ec179970-346d-4627-b2df-ed44637f91f8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16165" uid: bb50e861-859f-4c0c-a0fc-97cc69a9cfc1 kind: Event lastTimestamp: "2026-09-17T16:09:16Z" message: InferenceService [error-404-isvc-4f67a] is Ready metadata: creationTimestamp: "2026-09-17T16:09:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:09:16Z" name: error-404-isvc-4f67a.18d62785a0bdc143 namespace: kserve-ci-e2e-test resourceVersion: "16166" uid: c413e1f2-f2df-4273-97da-5cf088ee0a69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:28:39.270867Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25641" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-6d5c6-predictor-7555cb569-cqrlb to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d6289448f8a1e6 namespace: kserve-ci-e2e-test resourceVersion: "25645" uid: 41fed3fc-c95d-43d5-afbb-7d0f048c3bb2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-6d5c6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d628945054062e namespace: kserve-ci-e2e-test resourceVersion: "25655" uid: 805100a8-f2ae-4dcf-9f09-c8656188f5c3 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25650" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d628948ee1aca6 namespace: kserve-ci-e2e-test resourceVersion: "25675" uid: bbb684fe-1be3-4272-aa63-47cd0de37eba reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d6289490a6d279 namespace: kserve-ci-e2e-test resourceVersion: "25680" uid: dc0fc70f-75a6-4a88-add0-0d461a358169 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d62894939f06be namespace: kserve-ci-e2e-test resourceVersion: "25681" uid: 6f875662-fce9-40be-bcbc-3ce02864343b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d628949410a3ad namespace: kserve-ci-e2e-test resourceVersion: "25682" uid: eddf5fec-019b-4456-82d3-fe983c98ed7c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d62894941b2653 namespace: kserve-ci-e2e-test resourceVersion: "25683" uid: 030bc1b3-a64f-4c44-baa1-93b26e5c9b41 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d6289496e5d243 namespace: kserve-ci-e2e-test resourceVersion: "25686" uid: bd1ec996-efcb-4482-9395-21869701d294 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d628949751e3b9 namespace: kserve-ci-e2e-test resourceVersion: "25687" uid: 276ed9b7-313b-4e8c-94c4-133d4d5997c1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:29:17Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:17Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d62894b36194d0 namespace: kserve-ci-e2e-test resourceVersion: "26049" uid: 827a1db2-b20e-4676-90a4-a06a5d96d724 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:37:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:37:54Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d629157716fa6f namespace: kserve-ci-e2e-test resourceVersion: "30010" uid: b1d0827a-0208-4ae8-b92a-dfb93b7525f1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb namespace: kserve-ci-e2e-test resourceVersion: "25642" uid: 78ef311d-80cb-4e4c-8664-7fc6ed4f6125 kind: Event lastTimestamp: "2026-09-17T16:37:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:37:54Z" name: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb.18d6291577178a1f namespace: kserve-ci-e2e-test resourceVersion: "30011" uid: b0ce8cb3-375c-4022-90db-fe75d814d3b5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-6d5c6-predictor-7555cb569 namespace: kserve-ci-e2e-test resourceVersion: "25633" uid: f22cf20b-3df9-4217-b727-14e1d70bacf9 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: 'Created pod: error-404-isvc-6d5c6-predictor-7555cb569-cqrlb' metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-6d5c6-predictor-7555cb569.18d628944875ad85 namespace: kserve-ci-e2e-test resourceVersion: "25643" uid: 7bc83a71-ba90-4ec1-b65b-95af095c8208 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25631" uid: ddda8138-6db7-4453-8a2f-7d2d9b4f56b3 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: Scaled up replica set error-404-isvc-6d5c6-predictor-7555cb569 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-6d5c6-predictor.18d6289445fdcb0f namespace: kserve-ci-e2e-test resourceVersion: "25635" uid: d3a44909-f9e6-4aab-ab3b-a280d5d4dccf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: f350546d-8a13-40f1-9e9b-ceeba0e8be25 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:09Z" name: error-404-isvc-6d5c6-predictor.18d62897c47eb160 namespace: kserve-ci-e2e-test resourceVersion: "25980" uid: 163b7a9d-2f80-4d7b-92e6-3e6f89c40d11 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25632" uid: f350546d-8a13-40f1-9e9b-ceeba0e8be25 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:09Z" name: error-404-isvc-6d5c6-predictor.18d62897c55dadfd namespace: kserve-ci-e2e-test resourceVersion: "25981" uid: c40d1a53-7813-4846-a7ce-0dd452a0d100 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:29:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: f350546d-8a13-40f1-9e9b-ceeba0e8be25 kind: Event lastTimestamp: "2026-09-17T16:29:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:54Z" name: error-404-isvc-6d5c6-predictor.18d6289ec3685093 namespace: kserve-ci-e2e-test resourceVersion: "26383" uid: 8ccb1c68-e661-40fd-b5a3-44f6fb323494 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:29:24Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25816" uid: f350546d-8a13-40f1-9e9b-ceeba0e8be25 kind: Event lastTimestamp: "2026-09-17T16:29:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:29:24Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:54Z" name: error-404-isvc-6d5c6-predictor.18d6289ec41e34cb namespace: kserve-ci-e2e-test resourceVersion: "26384" uid: ad52ccd2-4e2b-4254-b078-01f04422338d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "25618" uid: f091fd58-2b58-4374-bef0-0af1e1f9ce83 kind: Event lastTimestamp: "2026-09-17T16:29:27Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:27Z" name: error-404-isvc-6d5c6.18d6289445f44b2e namespace: kserve-ci-e2e-test resourceVersion: "26138" uid: 93973cd3-bbdb-4cef-8597-a6c89c2fd329 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:27Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26136" uid: f091fd58-2b58-4374-bef0-0af1e1f9ce83 kind: Event lastTimestamp: "2026-09-17T16:29:27Z" message: InferenceService [error-404-isvc-6d5c6] is Ready metadata: creationTimestamp: "2026-09-17T16:29:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:27Z" name: error-404-isvc-6d5c6.18d6289f704f7f66 namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: bce69fe9-b07e-4f6d-99a8-f2287486d648 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:17:03.157075Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19494" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f2355b1338 namespace: kserve-ci-e2e-test resourceVersion: "19498" uid: c3be8961-9f22-4bab-aafd-8349fa53b917 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7c0d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f240d391c5 namespace: kserve-ci-e2e-test resourceVersion: "19513" uid: d6e4763a-7aa7-48b6-bb68-813901167692 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19504" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f27b43599c namespace: kserve-ci-e2e-test resourceVersion: "19530" uid: e6b75024-9390-40cf-81d0-03e14f227fb2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f27da52e29 namespace: kserve-ci-e2e-test resourceVersion: "19534" uid: 31444243-9d9b-48e8-be16-130fd59b1357 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f280a9abec namespace: kserve-ci-e2e-test resourceVersion: "19535" uid: dfe21068-d719-438c-92c7-9e35fd9141c5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f2810ba630 namespace: kserve-ci-e2e-test resourceVersion: "19536" uid: 73ab7d69-b742-4ce6-bbde-f937f107aff7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f281146142 namespace: kserve-ci-e2e-test resourceVersion: "19537" uid: d9e5262c-a137-4780-abe2-3cc5dac87158 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f283db2cf6 namespace: kserve-ci-e2e-test resourceVersion: "19538" uid: 67109ace-0ae8-4f52-a828-9e493ee2ff66 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f2843aca0a namespace: kserve-ci-e2e-test resourceVersion: "19539" uid: 3f536ccc-30ef-4e11-8394-65b613ff9909 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:17:40Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:40Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d627f29d9369f1 namespace: kserve-ci-e2e-test resourceVersion: "19835" uid: 49ca87cd-30cd-4247-934b-cb5fc0a11f10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d6280290ff00f1 namespace: kserve-ci-e2e-test resourceVersion: "20347" uid: 97b45ed4-f8da-49b4-83bc-01290267bdc9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d6280290ff882c namespace: kserve-ci-e2e-test resourceVersion: "20350" uid: b983394d-9f43-4157-af17-a389a78168e5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9 namespace: kserve-ci-e2e-test resourceVersion: "19497" uid: 725717d9-9089-4815-a226-4e44a3a09d88 kind: Event lastTimestamp: "2026-09-17T16:18:15Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:15Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9.18d6280325b64185 namespace: kserve-ci-e2e-test resourceVersion: "20458" uid: a412aef3-fefe-4d90-ae21-778576556c04 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7c0d2-predictor-dbcf8947c namespace: kserve-ci-e2e-test resourceVersion: "19487" uid: f9dbd46b-7a97-4f69-9ac3-82732716d756 kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'Created pod: error-404-isvc-7c0d2-predictor-dbcf8947c-cd8k9' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2-predictor-dbcf8947c.18d627f2348e4168 namespace: kserve-ci-e2e-test resourceVersion: "19495" uid: 1f3e0831-5e66-4913-9963-2ab843392aaa reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19484" uid: ebf6e897-cedd-49d4-a605-fd9f17aa08e1 kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: Scaled up replica set error-404-isvc-7c0d2-predictor-dbcf8947c from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2-predictor.18d627f23268ea62 namespace: kserve-ci-e2e-test resourceVersion: "19489" uid: 8ce7d172-5db7-4288-9af3-d42f2b6217c4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: ef48da90-e7b1-42f9-9cf4-bbe9b7f80cd7 kind: Event lastTimestamp: "2026-09-17T16:17:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:18Z" name: error-404-isvc-7c0d2-predictor.18d627f5b18b3b03 namespace: kserve-ci-e2e-test resourceVersion: "19661" uid: b9a862dc-7fa1-4d93-9aeb-a2118b917a24 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:18Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19486" uid: ef48da90-e7b1-42f9-9cf4-bbe9b7f80cd7 kind: Event lastTimestamp: "2026-09-17T16:17:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:18Z" name: error-404-isvc-7c0d2-predictor.18d627f5b23b44cc namespace: kserve-ci-e2e-test resourceVersion: "19663" uid: d696e1b8-8ba1-4a2c-8eea-8887a6a5ee55 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19662" uid: ef48da90-e7b1-42f9-9cf4-bbe9b7f80cd7 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: error-404-isvc-7c0d2-predictor.18d627f931902b81 namespace: kserve-ci-e2e-test resourceVersion: "20204" uid: 194762a9-cad9-4941-af3a-4ca999614b09 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:17:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19662" uid: ef48da90-e7b1-42f9-9cf4-bbe9b7f80cd7 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:17:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: error-404-isvc-7c0d2-predictor.18d627f93251d3b5 namespace: kserve-ci-e2e-test resourceVersion: "20205" uid: 8804814f-1b93-47c1-a41e-3ac4a96a6b28 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 198d2670-e2a6-4cbd-bf41-d7ed67d6fb81 kind: Event lastTimestamp: "2026-09-17T16:17:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:51Z" name: error-404-isvc-7c0d2.18d627f2326e2bdf namespace: kserve-ci-e2e-test resourceVersion: "20075" uid: 6e95e8de-9e30-4703-8233-798cb2063a4d reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 198d2670-e2a6-4cbd-bf41-d7ed67d6fb81 kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'Failed to update status for InferenceService "error-404-isvc-7c0d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2.18d627f233092242 namespace: kserve-ci-e2e-test resourceVersion: "19491" uid: a37a8da9-11db-456e-b35e-18c032339b6d reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19453" uid: 198d2670-e2a6-4cbd-bf41-d7ed67d6fb81 kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:03Z" name: error-404-isvc-7c0d2.18d627f233098163 namespace: kserve-ci-e2e-test resourceVersion: "19492" uid: 23def960-24a7-4ad7-a0bc-a3bd5311ce54 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "20071" uid: 198d2670-e2a6-4cbd-bf41-d7ed67d6fb81 kind: Event lastTimestamp: "2026-09-17T16:17:51Z" message: InferenceService [error-404-isvc-7c0d2] is Ready metadata: creationTimestamp: "2026-09-17T16:17:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:51Z" name: error-404-isvc-7c0d2.18d627fd5cc9b3c2 namespace: kserve-ci-e2e-test resourceVersion: "20072" uid: d8560b38-c85c-4515-8926-57c2e8ae4e71 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:27:28.949047Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24742" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s to ip-10-0-143-187.ec2.internal metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:27:28Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d62883e976ead4 namespace: kserve-ci-e2e-test resourceVersion: "24746" uid: d2b8e8d8-b093-4555-bde2-25830e858276 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-7ce9a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:29Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d62883f4b86374 namespace: kserve-ci-e2e-test resourceVersion: "24759" uid: ceebac11-0c34-48fe-a785-fa9d723f639b reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24752" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628842f36b39b namespace: kserve-ci-e2e-test resourceVersion: "24777" uid: fda823bc-f649-4238-999b-6902219d202b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628843105231f namespace: kserve-ci-e2e-test resourceVersion: "24780" uid: 4629a787-4785-464f-a215-58c559b3d13e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d62884342513a9 namespace: kserve-ci-e2e-test resourceVersion: "24781" uid: 11460ed7-1c8c-4d47-b144-aa3f0ff5a800 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628843498d4ce namespace: kserve-ci-e2e-test resourceVersion: "24782" uid: 797c1f0e-03ec-49e9-bf6a-db5843df0170 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d6288434a456d0 namespace: kserve-ci-e2e-test resourceVersion: "24783" uid: 84bd74a4-452e-49d9-af5d-b972f0425d87 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d6288437a99c0e namespace: kserve-ci-e2e-test resourceVersion: "24785" uid: f24bca50-c558-4f94-8556-30b0a63294d7 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628843819a7e1 namespace: kserve-ci-e2e-test resourceVersion: "24786" uid: f2fc3adc-4eb4-449a-9deb-d7902a177c83 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:28:07Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:07Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628848efee83c namespace: kserve-ci-e2e-test resourceVersion: "25086" uid: dceb1035-3e04-481e-bd5d-111ab0c98b4a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628943b546744 namespace: kserve-ci-e2e-test resourceVersion: "25600" uid: 1e67f809-134d-4a0d-bea1-0c8262df8167 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:39Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d628943b556b5d namespace: kserve-ci-e2e-test resourceVersion: "25602" uid: d55e3817-95f6-41f8-82a3-932380be25f5 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s namespace: kserve-ci-e2e-test resourceVersion: "24744" uid: 78af82a4-b903-4ebd-8908-57f189e04659 kind: Event lastTimestamp: "2026-09-17T16:28:42Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:42Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s.18d6289517473c53 namespace: kserve-ci-e2e-test resourceVersion: "25714" uid: 28c1868b-deaa-4eea-a0ae-848e8a2360a8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-143-187.ec2.internal source: component: kubelet host: ip-10-0-143-187.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7ce9a-predictor-5457d7f87f namespace: kserve-ci-e2e-test resourceVersion: "24734" uid: d16d3880-5ee7-45c4-b975-2bdce80974d1 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: 'Created pod: error-404-isvc-7ce9a-predictor-5457d7f87f-c7x4s' metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:28Z" name: error-404-isvc-7ce9a-predictor-5457d7f87f.18d62883e8c3922c namespace: kserve-ci-e2e-test resourceVersion: "24743" uid: 659ad5b5-7543-4bba-bfd8-a2a473193895 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24732" uid: ac6450c3-e1c3-4db5-b572-383cec23daa7 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Scaled up replica set error-404-isvc-7ce9a-predictor-5457d7f87f from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:28Z" name: error-404-isvc-7ce9a-predictor.18d62883e3dc00d7 namespace: kserve-ci-e2e-test resourceVersion: "24735" uid: 34e37671-3740-4b39-bd2e-0135d35f0471 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: 746006cd-8ebc-4a81-8f0c-0decc86e5171 kind: Event lastTimestamp: "2026-09-17T16:28:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:13Z" name: error-404-isvc-7ce9a-predictor.18d6288763788b20 namespace: kserve-ci-e2e-test resourceVersion: "25284" uid: b97100dc-ed2f-4628-9cfd-5022fe0c2877 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24733" uid: 746006cd-8ebc-4a81-8f0c-0decc86e5171 kind: Event lastTimestamp: "2026-09-17T16:28:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:13Z" name: error-404-isvc-7ce9a-predictor.18d62887645f600b namespace: kserve-ci-e2e-test resourceVersion: "25285" uid: 8acc8446-b050-47d7-b55e-b8c0b7a3a00b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 746006cd-8ebc-4a81-8f0c-0decc86e5171 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: error-404-isvc-7ce9a-predictor.18d6288ae342f6d8 namespace: kserve-ci-e2e-test resourceVersion: "25448" uid: 42c79b41-be86-46e8-8961-18ef5c1d6ceb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24909" uid: 746006cd-8ebc-4a81-8f0c-0decc86e5171 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: error-404-isvc-7ce9a-predictor.18d6288ae3c0699b namespace: kserve-ci-e2e-test resourceVersion: "25449" uid: 1fa9933a-83d1-41f2-80d7-f733386b601c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "24718" uid: a3be7937-93c1-4edb-a5fc-9991fc567cfb kind: Event lastTimestamp: "2026-09-17T16:28:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:17Z" name: error-404-isvc-7ce9a.18d62883e47e3c21 namespace: kserve-ci-e2e-test resourceVersion: "25329" uid: d3f0d750-f0df-4d1a-9aa4-49825cdbca6b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "25323" uid: a3be7937-93c1-4edb-a5fc-9991fc567cfb kind: Event lastTimestamp: "2026-09-17T16:28:17Z" message: InferenceService [error-404-isvc-7ce9a] is Ready metadata: creationTimestamp: "2026-09-17T16:28:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:17Z" name: error-404-isvc-7ce9a.18d6288f4cdd8f53 namespace: kserve-ci-e2e-test resourceVersion: "25324" uid: d3bf8815-95df-4c0a-8207-d954b4d85189 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:05:57.852903Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13867" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:05:57Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d627574e23112b namespace: kserve-ci-e2e-test resourceVersion: "13874" uid: 24be9e5b-6503-4a0a-bf9c-9c61db5d86e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275758da0dde namespace: kserve-ci-e2e-test resourceVersion: "13901" uid: 0c530fbb-96ec-4a0a-bc17-f45d43c7b4f2 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13883" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d627579396e4a2 namespace: kserve-ci-e2e-test resourceVersion: "14009" uid: a3e86f11-9a4c-4bc1-8dbb-fbbffac78a34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3c9053358f4938ace95cb81792a4ceb45b7d66d6fd701108f007bf1e1a9e783f" already present on machine metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275795bb096d namespace: kserve-ci-e2e-test resourceVersion: "14012" uid: a778e3d5-2b05-4d2b-a31b-d251f7e0e6f9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d627579990ffb0 namespace: kserve-ci-e2e-test resourceVersion: "14014" uid: 7fc8f970-bf22-4451-ad2a-1e56a9209e6b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275799f2bd1c namespace: kserve-ci-e2e-test resourceVersion: "14016" uid: 0de35f14-0ca6-4b4a-be3f-497af2c813b2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1941" metadata: creationTimestamp: "2026-09-17T16:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:04Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d62758e03af456 namespace: kserve-ci-e2e-test resourceVersion: "14087" uid: 22865d65-57a0-40b4-886c-011ce91256c7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1941" in 21.817s (21.817s including waiting). Image size: 1583199637 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275df4aec4d5 namespace: kserve-ci-e2e-test resourceVersion: "14280" uid: 521b35ca-0628-4f11-a3b9-6737a37c7ef2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275dfc7dbcd9 namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: f59e0d80-0603-4e87-8c27-eef0017ced27 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275dfd9d3b77 namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: 27451514-d066-4f85-b456-145890e40d5e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275e08cd7f62 namespace: kserve-ci-e2e-test resourceVersion: "14290" uid: 0f98002d-203d-486b-86c4-7e12ae54f726 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.048s (3.048s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275ebe867d56 namespace: kserve-ci-e2e-test resourceVersion: "14326" uid: 034f9d3a-db8d-45bb-a671-0b450565555b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275ec2177dae namespace: kserve-ci-e2e-test resourceVersion: "14330" uid: 5b211421-1785-418a-8d74-de4392e1c740 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275ec28d15ac namespace: kserve-ci-e2e-test resourceVersion: "14332" uid: 879501c6-5214-457d-afb2-5bb65ac7808d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 11 eventTime: null firstTimestamp: "2026-09-17T16:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.22:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6275ef74ba008 namespace: kserve-ci-e2e-test resourceVersion: "15714" uid: ead43360-66af-420a-a7a0-5e33cfd1431d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6277a7109f7a5 namespace: kserve-ci-e2e-test resourceVersion: "15618" uid: 87e69bce-afae-4892-805d-7ca9e0e2c43f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6277a710a9661 namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: acd63bd0-7a6a-4f0b-89f5-734d9ac89d58 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf namespace: kserve-ci-e2e-test resourceVersion: "13871" uid: eb7c0de5-24a4-4e31-a1e1-a50300dae572 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Readiness probe failed: Get "https://10.133.0.22:8643/healthz": dial tcp 10.133.0.22:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf.18d6277b2410a47d namespace: kserve-ci-e2e-test resourceVersion: "15734" uid: 84c451a5-2554-4fc9-a70a-00ed99a326ac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:57Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-5fdd7b49c5 namespace: kserve-ci-e2e-test resourceVersion: "13852" uid: 68680570-4f05-4f30-b0fc-29dd3adcdb0f kind: Event lastTimestamp: "2026-09-17T16:05:57Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-5fdd7b49c5-tcqdf' metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:57Z" name: isvc-sklearn-graph-1-predictor-5fdd7b49c5.18d627574d668ec7 namespace: kserve-ci-e2e-test resourceVersion: "13869" uid: e68f3d00-280d-4d2d-a9b0-ca97f4849d4c reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13850" uid: 5f7ed1f3-f33f-4f6f-8846-65c8e01d95e6 kind: Event lastTimestamp: "2026-09-17T16:05:57Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-5fdd7b49c5 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:57Z" name: isvc-sklearn-graph-1-predictor.18d6275740f4d88d namespace: kserve-ci-e2e-test resourceVersion: "13854" uid: 5e2f13c3-c26d-4b85-a868-a57fc341ae8c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: 415fb415-16c1-47a0-bea6-8aa7bc764a9b kind: Event lastTimestamp: "2026-09-17T16:06:57Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:57Z" name: isvc-sklearn-graph-1-predictor.18d6275ac8c073f9 namespace: kserve-ci-e2e-test resourceVersion: "14571" uid: 39aca08c-69bd-4fed-9d13-ad7cd43e8e0c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "13859" uid: 415fb415-16c1-47a0-bea6-8aa7bc764a9b kind: Event lastTimestamp: "2026-09-17T16:06:57Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:57Z" name: isvc-sklearn-graph-1-predictor.18d6275acaa2a74b namespace: kserve-ci-e2e-test resourceVersion: "14572" uid: 23ed6af3-ca4b-4a98-8c79-dca26af320cc reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 415fb415-16c1-47a0-bea6-8aa7bc764a9b kind: Event lastTimestamp: "2026-09-17T16:08:12Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:12Z" name: isvc-sklearn-graph-1-predictor.18d62768c7a03965 namespace: kserve-ci-e2e-test resourceVersion: "15389" uid: c4f19761-58ee-445c-a7f7-9aeeb5876eae reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:07:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14159" uid: 415fb415-16c1-47a0-bea6-8aa7bc764a9b kind: Event lastTimestamp: "2026-09-17T16:08:12Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:12Z" name: isvc-sklearn-graph-1-predictor.18d62768c8264f22 namespace: kserve-ci-e2e-test resourceVersion: "15390" uid: db8400a6-9ec0-4c12-bd6a-a183573e403b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T16:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "13839" uid: 7d396b11-c9f0-4437-93fb-49fc124a2f08 kind: Event lastTimestamp: "2026-09-17T16:07:51Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:51Z" name: isvc-sklearn-graph-1.18d62757486d13c9 namespace: kserve-ci-e2e-test resourceVersion: "15217" uid: 20ec9072-59b0-4e4b-914f-33c9ed8c6beb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 7d396b11-c9f0-4437-93fb-49fc124a2f08 kind: Event lastTimestamp: "2026-09-17T16:07:51Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-09-17T16:07:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:51Z" name: isvc-sklearn-graph-1.18d62771af9978f9 namespace: kserve-ci-e2e-test resourceVersion: "15215" uid: c07b3e5e-3263-41f0-9cf3-81403fa75542 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:05:58.685305Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13982" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:05:58Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d627577fc094a0 namespace: kserve-ci-e2e-test resourceVersion: "13986" uid: bb06e686-9b39-4229-830f-0d8953c1385c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275788e77c8e namespace: kserve-ci-e2e-test resourceVersion: "13998" uid: 6d04d8c1-e456-432b-9a2d-e64cba90fced reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13990" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d62757c91e4f67 namespace: kserve-ci-e2e-test resourceVersion: "14038" uid: b9c41c10-d516-4fd5-9c5b-8ea9e7b833f9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:59Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:05:59Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3c9053358f4938ace95cb81792a4ceb45b7d66d6fd701108f007bf1e1a9e783f" already present on machine metadata: creationTimestamp: "2026-09-17T16:05:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:59Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d62757cb50b049 namespace: kserve-ci-e2e-test resourceVersion: "14040" uid: 720b1b7e-d6e0-42d8-b277-4413cd9da2e7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:00Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:00Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d62757cf9aa793 namespace: kserve-ci-e2e-test resourceVersion: "14041" uid: a6f0863e-dc93-4a03-a5d7-abbb703bd50a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:00Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:00Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T16:06:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:00Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d62757d06d8df2 namespace: kserve-ci-e2e-test resourceVersion: "14043" uid: 1a39bba9-f547-4696-897b-2a230337f698 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:04Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1941" metadata: creationTimestamp: "2026-09-17T16:06:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:04Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d62758e061f837 namespace: kserve-ci-e2e-test resourceVersion: "14088" uid: 25f80701-a076-4059-bbe0-dc75026c4174 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1941" in 21.814s (21.814s including waiting). Image size: 1583199637 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275df4a7d48b namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: 9f07ce10-1a1c-4620-8a94-71da851d77dc reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275dfc7cb643 namespace: kserve-ci-e2e-test resourceVersion: "14285" uid: f45290e3-f9c0-46bd-95ac-8ed0be950cdc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275dfe0b800b namespace: kserve-ci-e2e-test resourceVersion: "14288" uid: 39485ff8-b501-4abb-b819-ad1bde9f8180 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275e08cd91d0 namespace: kserve-ci-e2e-test resourceVersion: "14291" uid: 966e448b-c7c8-4a34-9ddc-b63cf5899ecb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.133s (3.133s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275ec38bd8f9 namespace: kserve-ci-e2e-test resourceVersion: "14334" uid: 0c9f075b-ec82-4674-94e1-87cc0eb01dc2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275ec640b682 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 14de52c9-26fb-46d7-bdd6-ea7c9c5103df reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275ec69f49fc namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: 2bbb1027-52fb-44d3-8635-6af16657cebc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T16:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:07:36Z" message: 'Readiness probe failed: dial tcp 10.133.0.25:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:36Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6275ef783019f namespace: kserve-ci-e2e-test resourceVersion: "14895" uid: 6bfae914-2ebd-4dd6-bfb6-7075024d8d88 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6277a6cfc5ae2 namespace: kserve-ci-e2e-test resourceVersion: "15601" uid: f8e111e6-4eee-4c96-bf4e-9dc15c67bf07 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6277a6cfd311f namespace: kserve-ci-e2e-test resourceVersion: "15603" uid: 6c01906a-ec8d-4a6f-b73b-1ed3d511d121 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9 namespace: kserve-ci-e2e-test resourceVersion: "13984" uid: 28463381-bdb3-4adf-9f12-e78f3b66abb1 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Readiness probe failed: Get "https://10.133.0.25:8643/healthz": dial tcp 10.133.0.25:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9.18d6277b241153c0 namespace: kserve-ci-e2e-test resourceVersion: "15735" uid: ccdc2000-a0f1-4880-ab2f-2671f5f2d88e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-7c5995c664 namespace: kserve-ci-e2e-test resourceVersion: "13968" uid: 0d01c5be-0c5a-4d78-990b-9e5ee9aaa3a1 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-7c5995c664-t9nq9' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: isvc-sklearn-graph-2-predictor-7c5995c664.18d627577f3662d0 namespace: kserve-ci-e2e-test resourceVersion: "13983" uid: 9544e11a-305e-4c6c-a788-94db0ba44b9b reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13966" uid: d6ec979f-f78f-4898-984d-049526ef0b8e kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-7c5995c664 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: isvc-sklearn-graph-2-predictor.18d62757796061d3 namespace: kserve-ci-e2e-test resourceVersion: "13969" uid: 097c75ce-5dfc-4180-9cb9-82129075a234 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 408c751b-e6f4-4c0e-87c3-57c95145e6af kind: Event lastTimestamp: "2026-09-17T16:06:58Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:58Z" name: isvc-sklearn-graph-2-predictor.18d6275af814add2 namespace: kserve-ci-e2e-test resourceVersion: "14589" uid: b85c8876-db9e-4ce9-be2b-3aac07a8b219 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "13967" uid: 408c751b-e6f4-4c0e-87c3-57c95145e6af kind: Event lastTimestamp: "2026-09-17T16:06:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:58Z" name: isvc-sklearn-graph-2-predictor.18d6275af86bf1bd namespace: kserve-ci-e2e-test resourceVersion: "14590" uid: 11f946a9-b21c-4be4-ba83-5df121d7e5d2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 408c751b-e6f4-4c0e-87c3-57c95145e6af kind: Event lastTimestamp: "2026-09-17T16:08:13Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:13Z" name: isvc-sklearn-graph-2-predictor.18d62768f5518124 namespace: kserve-ci-e2e-test resourceVersion: "15397" uid: ce63969f-c6c5-406c-80b9-3c75192f15e8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:07:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14175" uid: 408c751b-e6f4-4c0e-87c3-57c95145e6af kind: Event lastTimestamp: "2026-09-17T16:08:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:13Z" name: isvc-sklearn-graph-2-predictor.18d62768f5b18318 namespace: kserve-ci-e2e-test resourceVersion: "15398" uid: 7c369198-aa93-4c56-9f44-85abc385670f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "13959" uid: 1dcbf0c5-1e04-444a-bdcb-d2e7f08163c4 kind: Event lastTimestamp: "2026-09-17T16:07:46Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:46Z" name: isvc-sklearn-graph-2.18d627577975a805 namespace: kserve-ci-e2e-test resourceVersion: "15023" uid: 1ce43b9f-d013-4052-9348-031e41c7f8da reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:46Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15020" uid: 1dcbf0c5-1e04-444a-bdcb-d2e7f08163c4 kind: Event lastTimestamp: "2026-09-17T16:07:46Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-09-17T16:07:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:46Z" name: isvc-sklearn-graph-2.18d627708412a92d namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: bb1aba7e-2749-4424-b32b-69460b7b40af reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:05:58.255109Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13923" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-hq8gt to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d62757661c3bab namespace: kserve-ci-e2e-test resourceVersion: "13927" uid: 19c9a255-3758-4a63-8e8f-20cb47d7dc1b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13938" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275787d7de77 namespace: kserve-ci-e2e-test resourceVersion: "13995" uid: 97f336fd-0449-454b-b9ac-5571744567e7 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:3c9053358f4938ace95cb81792a4ceb45b7d66d6fd701108f007bf1e1a9e783f" already present on machine metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275789fd6afa namespace: kserve-ci-e2e-test resourceVersion: "13999" uid: 037852de-8847-487c-8a7f-da66754a3e58 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d627578c7c3e02 namespace: kserve-ci-e2e-test resourceVersion: "14002" uid: b78cd1f5-1e4b-4424-b5b1-a7b73303103b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d627578cd80a3b namespace: kserve-ci-e2e-test resourceVersion: "14003" uid: 019b1407-3032-40c6-90fc-ae71b9100e96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:03Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-09-17T16:06:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:03Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d62758a40e6608 namespace: kserve-ci-e2e-test resourceVersion: "14074" uid: c424fe41-9afa-4288-9508-2966f3eb471f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:27Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 23.632s (23.632s including waiting). Image size: 1435742979 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275e24a3f4e0 namespace: kserve-ci-e2e-test resourceVersion: "14297" uid: 3ef14f6f-c0ac-4757-b6f8-73b60506a510 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275e2be7951d namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: ac042d8a-9e34-434a-a58d-99390581c899 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275e2e01fa4f namespace: kserve-ci-e2e-test resourceVersion: "14299" uid: 560389ee-139f-43d9-a66b-7643d5dcce8e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:27Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T16:06:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:27Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275e2e22fc37 namespace: kserve-ci-e2e-test resourceVersion: "14300" uid: c265e717-1f31-4240-8911-9ab120801885 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.426s (2.426s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275ebecc0638 namespace: kserve-ci-e2e-test resourceVersion: "14327" uid: 975244fa-a03e-49eb-ac02-a5346768ab36 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275ec2731ec5 namespace: kserve-ci-e2e-test resourceVersion: "14331" uid: 02c3d5e8-346d-4ac5-8d95-09ea6e5a4d68 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275ec2d966d5 namespace: kserve-ci-e2e-test resourceVersion: "14333" uid: 72c17f7c-c3d5-491e-b5be-52cdafb8c850 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-09-17T16:06:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:07:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.24:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:06:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:26Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6275ef7a3bbac namespace: kserve-ci-e2e-test resourceVersion: "14816" uid: 5c5b76ff-3960-4925-b827-3f0d6314119a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6277a75e50bd9 namespace: kserve-ci-e2e-test resourceVersion: "15630" uid: 0df23a6e-5a18-4011-b154-06db6b499413 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6277a75e58649 namespace: kserve-ci-e2e-test resourceVersion: "15631" uid: 9d8b0d37-8a07-4e6d-92ac-4ecc7a4510c7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt namespace: kserve-ci-e2e-test resourceVersion: "13925" uid: a7a58a8f-f509-4f53-a4b0-e07c086a2e3e kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Readiness probe failed: Get "https://10.133.0.24:8643/healthz": dial tcp 10.133.0.24:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: isvc-xgboost-graph-predictor-555fc69d56-hq8gt.18d6277b2411bc49 namespace: kserve-ci-e2e-test resourceVersion: "15736" uid: 52960e4c-d34b-46c7-abac-e26d849cab5e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "13912" uid: e7bf7d5b-2d7d-46b0-982c-0c4ef424c779 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-hq8gt' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor-555fc69d56.18d6275765613e79 namespace: kserve-ci-e2e-test resourceVersion: "13924" uid: 16dffaa1-7767-43f8-aa1b-6654b4128ab2 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13909" uid: a98df3f6-07a7-458f-9859-4135bf9a617a kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: isvc-xgboost-graph-predictor.18d627575f55b2ab namespace: kserve-ci-e2e-test resourceVersion: "13914" uid: 0fe21e5d-6329-467d-9324-97910a53ae55 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: 343f410f-1d1b-4716-b657-779fb495cd4c kind: Event lastTimestamp: "2026-09-17T16:06:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:43Z" name: isvc-xgboost-graph-predictor.18d6275adf055d60 namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: b707a6cc-b85e-4d94-aa8e-c85addbfa01e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:06:13Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "13913" uid: 343f410f-1d1b-4716-b657-779fb495cd4c kind: Event lastTimestamp: "2026-09-17T16:06:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:06:43Z" name: isvc-xgboost-graph-predictor.18d6275adfd972e8 namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: 83c8f8d5-0e63-497c-9bdd-5d387a874a19 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 343f410f-1d1b-4716-b657-779fb495cd4c kind: Event lastTimestamp: "2026-09-17T16:07:58Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:58Z" name: isvc-xgboost-graph-predictor.18d627655d52b1e5 namespace: kserve-ci-e2e-test resourceVersion: "15275" uid: c347cbd3-fcef-411f-a220-0d3c2061a7c7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:06:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14168" uid: 343f410f-1d1b-4716-b657-779fb495cd4c kind: Event lastTimestamp: "2026-09-17T16:07:58Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:06:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:58Z" name: isvc-xgboost-graph-predictor.18d627655e117498 namespace: kserve-ci-e2e-test resourceVersion: "15276" uid: dbf413c1-14b6-43f6-aa9c-dd6eac29c5da reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "13893" uid: eb6e5617-0345-4dce-bafd-03f2871a60ad kind: Event lastTimestamp: "2026-09-17T16:07:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:37Z" name: isvc-xgboost-graph.18d627575fdbe6eb namespace: kserve-ci-e2e-test resourceVersion: "14907" uid: e5705f5d-2af9-47a2-ae61-97beb67ce3d5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: eb6e5617-0345-4dce-bafd-03f2871a60ad kind: Event lastTimestamp: "2026-09-17T16:07:36Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-09-17T16:07:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:36Z" name: isvc-xgboost-graph.18d6276e60301743 namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: 5515e1bd-4de8-4267-860d-f062012e6451 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:08:17.776873Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15442" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-7b8d5555b8-jkz7p to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:08:17Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:08:17Z" name: model-chainer-7b8d5555b8-jkz7p.18d62777e2417430 namespace: kserve-ci-e2e-test resourceVersion: "15447" uid: 9ca2a893-3097-4326-9fc9-bc3ff7754b42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:17Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:17Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:17Z" name: model-chainer-7b8d5555b8-jkz7p.18d62777eea130ff namespace: kserve-ci-e2e-test resourceVersion: "15458" uid: b22ea54b-5144-4402-9d9f-31fc724c653f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:18Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:18Z" message: Add eth0 [10.132.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:08:18Z" name: model-chainer-7b8d5555b8-jkz7p.18d6277827f039f4 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 73676d2a-6bca-48b0-bc6b-b5a0fadfef0f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:18Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:18Z" name: model-chainer-7b8d5555b8-jkz7p.18d627782a04e6ae namespace: kserve-ci-e2e-test resourceVersion: "15478" uid: 496aacda-c821-4014-bd8e-189009128436 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:19Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-09-17T16:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:19Z" name: model-chainer-7b8d5555b8-jkz7p.18d627782ceebee7 namespace: kserve-ci-e2e-test resourceVersion: "15480" uid: 291ea489-e19e-43e3-bb92-60789f24de30 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:19Z" message: Started container model-chainer metadata: creationTimestamp: "2026-09-17T16:08:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:19Z" name: model-chainer-7b8d5555b8-jkz7p.18d627782d56257f namespace: kserve-ci-e2e-test resourceVersion: "15481" uid: 6b7cb64d-e61a-4f61-bb63-0aaae9a80cda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:28Z" name: model-chainer-7b8d5555b8-jkz7p.18d6277a61f42116 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: c974eb1b-5198-4e18-9caf-dd652de689ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:08:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-7b8d5555b8-jkz7p namespace: kserve-ci-e2e-test resourceVersion: "15446" uid: d0249e71-f1a4-4d5a-a487-2e8cc254cddd kind: Event lastTimestamp: "2026-09-17T16:08:55Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:55Z" name: model-chainer-7b8d5555b8-jkz7p.18d6277adccca19d namespace: kserve-ci-e2e-test resourceVersion: "15987" uid: 2c2a6fb3-7946-499a-b987-40a0b3f22c26 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:17Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-7b8d5555b8 namespace: kserve-ci-e2e-test resourceVersion: "15438" uid: b5c01852-b56e-4baa-bc72-2790cd6de64e kind: Event lastTimestamp: "2026-09-17T16:08:17Z" message: 'Created pod: model-chainer-7b8d5555b8-jkz7p' metadata: creationTimestamp: "2026-09-17T16:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:17Z" name: model-chainer-7b8d5555b8.18d62777e15d0006 namespace: kserve-ci-e2e-test resourceVersion: "15444" uid: 0dc7eee1-d29a-4b54-a752-06a0b288ad20 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:17Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15436" uid: 6fe03b59-6f3c-4b38-bbc5-d002a2167e9d kind: Event lastTimestamp: "2026-09-17T16:08:17Z" message: Scaled up replica set model-chainer-7b8d5555b8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:17Z" name: model-chainer.18d62777de29645c namespace: kserve-ci-e2e-test resourceVersion: "15440" uid: cc543855-8579-4e4f-85d2-c3fb45d62227 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:17Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15454" uid: 4c713fe9-414d-4929-91b1-b6b2caf8735a kind: Event lastTimestamp: "2026-09-17T16:08:17Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-09-17T16:08:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:17Z" name: model-chainer.18d62777eb645d10 namespace: kserve-ci-e2e-test resourceVersion: "15456" uid: 49c3f37a-7805-446d-a501-36df1ab16561 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:19:54.066980Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21518" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-26f37-55f4c6587d-gplb5 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a00615509 namespace: kserve-ci-e2e-test resourceVersion: "21524" uid: 942a2aee-b910-4010-a7b8-62bc11093181 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-26f37-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a0843c25c namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 11c019d2-b710-40b8-8d21-6658764fb169 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:55Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21528" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:19:55Z" message: Add eth0 [10.132.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:19:55Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a463407c1 namespace: kserve-ci-e2e-test resourceVersion: "21550" uid: 9df9dc79-c9fd-49de-bb1b-e7b4053b25d2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26f37} kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:19:55Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:55Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a47c2b9ee namespace: kserve-ci-e2e-test resourceVersion: "21552" uid: e9fdcd76-94ef-4f01-8e76-0ba1634fef2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26f37} kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:19:55Z" message: 'Created container: sequence-graph-26f37' metadata: creationTimestamp: "2026-09-17T16:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:55Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a4a6c7d2e namespace: kserve-ci-e2e-test resourceVersion: "21553" uid: 7aa83479-8ebf-4aa4-a248-575e7ce26c08 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26f37} kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:19:55Z" message: Started container sequence-graph-26f37 metadata: creationTimestamp: "2026-09-17T16:19:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:55Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6281a4adc25ec namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 82a691ab-ae83-4644-b4c4-0babefc3d2e1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26f37} kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Stopping container sequence-graph-26f37 metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:08Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6288d22b8bc93 namespace: kserve-ci-e2e-test resourceVersion: "25110" uid: 9ca51c48-47c4-4ab5-82c6-5c98b7b49875 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-26f37} kind: Pod name: sequence-graph-26f37-55f4c6587d-gplb5 namespace: kserve-ci-e2e-test resourceVersion: "21522" uid: 57386200-0320-4e87-bbde-b550f3fecd1e kind: Event lastTimestamp: "2026-09-17T16:28:37Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:28:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:37Z" name: sequence-graph-26f37-55f4c6587d-gplb5.18d6288dfd76d21e namespace: kserve-ci-e2e-test resourceVersion: "25525" uid: cedaacf2-ee8c-43f0-8e21-d8fe77334337 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-26f37-55f4c6587d namespace: kserve-ci-e2e-test resourceVersion: "21515" uid: 117a7406-0216-4213-90f4-fccfbce4ea87 kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: 'Created pod: sequence-graph-26f37-55f4c6587d-gplb5' metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37-55f4c6587d.18d62819ff37ba40 namespace: kserve-ci-e2e-test resourceVersion: "21520" uid: 8422318a-a23e-4412-826d-748efadb13cf reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: fb5152d0-2486-46cb-a0e9-7ebb85bda42a kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: Scaled up replica set sequence-graph-26f37-55f4c6587d from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37.18d62819fd8a2184 namespace: kserve-ci-e2e-test resourceVersion: "21516" uid: 6df9792e-4dda-4370-9174-e92751f749d0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21530" uid: 793d0cf4-49f7-4649-8788-999218622ee4 kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: InferenceGraph [sequence-graph-26f37] is Ready metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37.18d6281a078a9c2e namespace: kserve-ci-e2e-test resourceVersion: "21533" uid: e5e8c3a3-7f33-42d4-9428-0afea4fbc8a7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 793d0cf4-49f7-4649-8788-999218622ee4 kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: 'Failed to update status for InferenceGraph "sequence-graph-26f37": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37.18d6281a0cf68a70 namespace: kserve-ci-e2e-test resourceVersion: "21541" uid: 03521a10-9295-499f-a78a-3a5095bb4f3e reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:54Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21506" uid: 793d0cf4-49f7-4649-8788-999218622ee4 kind: Event lastTimestamp: "2026-09-17T16:19:54Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:19:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:54Z" name: sequence-graph-26f37.18d6281a0cf6d32d namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: ff524d81-f685-4f36-850b-5671baa7bc52 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:20:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: a380b137-104e-4038-8f38-93baff99e2e7 kind: Event lastTimestamp: "2026-09-17T16:20:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:20:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:24Z" name: sequence-graph-26f37.18d6281d7c11631e namespace: kserve-ci-e2e-test resourceVersion: "21765" uid: 98ff252e-cca7-4d8e-b863-a2b7e9bc5e45 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:20:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21514" uid: a380b137-104e-4038-8f38-93baff99e2e7 kind: Event lastTimestamp: "2026-09-17T16:20:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:20:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:24Z" name: sequence-graph-26f37.18d6281d7c910478 namespace: kserve-ci-e2e-test resourceVersion: "21766" uid: 9ae4d8cc-7172-47ee-8356-c0d8c0918b94 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:43.652455Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20713" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-48574-bd8ccdb8c-g8gxl to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:43Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d628099b58eec4 namespace: kserve-ci-e2e-test resourceVersion: "20717" uid: 83c6198c-78fd-42ab-9b25-84b0bcb0c4dc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:44Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20722" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:18:44Z" message: Add eth0 [10.132.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:44Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d62809bd2e696d namespace: kserve-ci-e2e-test resourceVersion: "20743" uid: e6aa6cda-be35-4360-b098-a581ce034d59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-48574} kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:18:44Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:44Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d62809c0254c10 namespace: kserve-ci-e2e-test resourceVersion: "20745" uid: cdcec8ed-798e-4c07-9032-b17bd3ce2f62 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-48574} kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:18:44Z" message: 'Created container: sequence-graph-48574' metadata: creationTimestamp: "2026-09-17T16:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:44Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d62809c30ff2e1 namespace: kserve-ci-e2e-test resourceVersion: "20746" uid: 1ca4178d-1b42-41c9-b038-5efc51ab1b80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-48574} kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:18:44Z" message: Started container sequence-graph-48574 metadata: creationTimestamp: "2026-09-17T16:18:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:44Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d62809c37c5f09 namespace: kserve-ci-e2e-test resourceVersion: "20747" uid: 342a683f-dbf8-4a40-988c-609067be7fc1 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-48574} kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Stopping container sequence-graph-48574 metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:53Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d6280befb3d99d namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: 52596f84-6bb1-40cd-ac7e-ef84013dd6e6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-48574} kind: Pod name: sequence-graph-48574-bd8ccdb8c-g8gxl namespace: kserve-ci-e2e-test resourceVersion: "20715" uid: bdf47d46-b8d6-493c-8891-89e143ddbca1 kind: Event lastTimestamp: "2026-09-17T16:19:20Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:20Z" name: sequence-graph-48574-bd8ccdb8c-g8gxl.18d6280c7906c5e7 namespace: kserve-ci-e2e-test resourceVersion: "21230" uid: b3a74b97-1979-44c0-985d-9f9752ac6a39 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-48574-bd8ccdb8c namespace: kserve-ci-e2e-test resourceVersion: "20708" uid: d46dc125-081a-4ad8-9bae-8559135f44c3 kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'Created pod: sequence-graph-48574-bd8ccdb8c-g8gxl' metadata: creationTimestamp: "2026-09-17T16:18:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: sequence-graph-48574-bd8ccdb8c.18d6280999fbd0f9 namespace: kserve-ci-e2e-test resourceVersion: "20714" uid: 73ec5502-cb7f-43e9-b178-7fd7dd479f08 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-48574 namespace: kserve-ci-e2e-test resourceVersion: "20706" uid: 8fcc3c0f-353e-4832-99b7-734bc5f777b9 kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: Scaled up replica set sequence-graph-48574-bd8ccdb8c from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: sequence-graph-48574.18d628099778f1cd namespace: kserve-ci-e2e-test resourceVersion: "20709" uid: 928e4d8d-6a85-42fc-924a-a6c55190b2a1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:43Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-48574 namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: 08dce13b-821b-4e19-a883-e41d091c0495 kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: InferenceGraph [sequence-graph-48574] is Ready metadata: creationTimestamp: "2026-09-17T16:18:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:43Z" name: sequence-graph-48574.18d62809ac3df29a namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: 4596b49b-0ed6-446e-a4ce-2920225bfa0e reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:09:28.758260Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-4f67a-5789f8f4f9-mpwtf to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:09:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:09:28Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d6278869136cd1 namespace: kserve-ci-e2e-test resourceVersion: "16273" uid: dfbdb284-8377-4c9a-a7ec-0b60f04bea79 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:29Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:09:29Z" message: Add eth0 [10.132.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:09:29Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d627888acd735f namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: 49526335-f7e2-453c-8430-e2430f7b06d9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4f67a} kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:09:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:09:29Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d627888ccf5953 namespace: kserve-ci-e2e-test resourceVersion: "16298" uid: 33946e15-5d86-414e-9c3f-e7f78a65b364 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4f67a} kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:09:29Z" message: 'Created container: sequence-graph-4f67a' metadata: creationTimestamp: "2026-09-17T16:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:09:29Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d627888faabb0e namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 31ff1bfb-a9a3-43e2-9b08-66968bcc6ea9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4f67a} kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:09:29Z" message: Started container sequence-graph-4f67a metadata: creationTimestamp: "2026-09-17T16:09:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:09:29Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d62788901c4524 namespace: kserve-ci-e2e-test resourceVersion: "16300" uid: eda7ab2c-21c4-423e-8b25-75dfd8528c5f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4f67a} kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Stopping container sequence-graph-4f67a metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d627fb8ad37339 namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: f9abc117-b0b8-4073-98be-03aaa3c6e7a8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-4f67a} kind: Pod name: sequence-graph-4f67a-5789f8f4f9-mpwtf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 8df92046-3771-4262-9328-c69141335e45 kind: Event lastTimestamp: "2026-09-17T16:18:10Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:10Z" name: sequence-graph-4f67a-5789f8f4f9-mpwtf.18d627fc25c4a08b namespace: kserve-ci-e2e-test resourceVersion: "20275" uid: 419445d3-f749-4dcd-8669-497c50d61731 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-4f67a-5789f8f4f9 namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: e64b7491-4862-462c-b5c1-025397c37d0d kind: Event lastTimestamp: "2026-09-17T16:09:28Z" message: 'Created pod: sequence-graph-4f67a-5789f8f4f9-mpwtf' metadata: creationTimestamp: "2026-09-17T16:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:28Z" name: sequence-graph-4f67a-5789f8f4f9.18d6278868913450 namespace: kserve-ci-e2e-test resourceVersion: "16269" uid: 52fe5450-77d4-454a-8207-48b46ad3a9a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16263" uid: 3a3a011e-a20a-41d7-bc56-8341fe93deda kind: Event lastTimestamp: "2026-09-17T16:09:28Z" message: Scaled up replica set sequence-graph-4f67a-5789f8f4f9 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:28Z" name: sequence-graph-4f67a.18d62788677145f9 namespace: kserve-ci-e2e-test resourceVersion: "16266" uid: 3b75f612-3d44-461b-9ca2-13eeebd93171 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16280" uid: c3e6a4d0-7eb9-4750-a09b-e17b9fdfca30 kind: Event lastTimestamp: "2026-09-17T16:09:28Z" message: InferenceGraph [sequence-graph-4f67a] is Ready metadata: creationTimestamp: "2026-09-17T16:09:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:09:28Z" name: sequence-graph-4f67a.18d627886f721c6e namespace: kserve-ci-e2e-test resourceVersion: "16282" uid: 4c820467-9685-4f48-8752-b17d2118cef0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:09:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 45169537-cc5d-4682-af8f-657375fdf485 kind: Event lastTimestamp: "2026-09-17T16:10:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:10:13Z" name: sequence-graph-4f67a.18d6278be623a6d9 namespace: kserve-ci-e2e-test resourceVersion: "16625" uid: 6b14e35b-391c-444e-910a-e870b3515a03 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:09:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 45169537-cc5d-4682-af8f-657375fdf485 kind: Event lastTimestamp: "2026-09-17T16:10:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:09:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:10:13Z" name: sequence-graph-4f67a.18d6278be7496899 namespace: kserve-ci-e2e-test resourceVersion: "16626" uid: 5e23cf97-7ad6-434c-b9d1-0f88479d639d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:29:39.144432Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26242" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-6d5c6-5d4986d579-ngwcr to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d628a239b6b958 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: b4440a87-de83-409d-8086-f440a34d9656 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26253" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: Add eth0 [10.132.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d628a25b8a8683 namespace: kserve-ci-e2e-test resourceVersion: "26267" uid: 5f0b9bb2-6feb-4a65-b33c-f74db62c9c34 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6d5c6} kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d628a25da3804a namespace: kserve-ci-e2e-test resourceVersion: "26269" uid: ddf8c306-62d0-4e71-b5ec-35619e5260a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6d5c6} kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: 'Created container: splitter-graph-6d5c6' metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d628a260bfdf89 namespace: kserve-ci-e2e-test resourceVersion: "26270" uid: 11a05049-bc98-41f4-93ac-a14608ddc7dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6d5c6} kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: Started container splitter-graph-6d5c6 metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d628a261301b51 namespace: kserve-ci-e2e-test resourceVersion: "26271" uid: 100df87c-2018-4113-9eee-9b3d7e282b9e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:37:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6d5c6} kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:37:53Z" message: Stopping container splitter-graph-6d5c6 metadata: creationTimestamp: "2026-09-17T16:37:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:37:53Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d6291566365e86 namespace: kserve-ci-e2e-test resourceVersion: "29977" uid: 90b71899-4776-4a79-b2ff-424e6a3a54cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:37:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-6d5c6} kind: Pod name: splitter-graph-6d5c6-5d4986d579-ngwcr namespace: kserve-ci-e2e-test resourceVersion: "26245" uid: 1d3edbeb-6c3d-4098-ac79-3820ea076531 kind: Event lastTimestamp: "2026-09-17T16:38:21Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:37:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:38:21Z" name: splitter-graph-6d5c6-5d4986d579-ngwcr.18d629160499f25e namespace: kserve-ci-e2e-test resourceVersion: "30209" uid: 9da44b16-8047-4cb4-bc47-98b2630dbd8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-6d5c6-5d4986d579 namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: e530b888-70b7-402b-b84f-06ecc27ce9e2 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: 'Created pod: splitter-graph-6d5c6-5d4986d579-ngwcr' metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6-5d4986d579.18d628a239415190 namespace: kserve-ci-e2e-test resourceVersion: "26246" uid: eacce2ca-e916-436b-b502-c5e5aa02d705 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26237" uid: 222e3d9d-bc89-4093-97f5-7936af3922cf kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: Scaled up replica set splitter-graph-6d5c6-5d4986d579 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6.18d628a2378437a7 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 505ae987-3d50-4545-8597-7cbac9f67e88 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:39Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26257" uid: b6fbd2ae-f2d8-40a4-8f62-7766d4c7e254 kind: Event lastTimestamp: "2026-09-17T16:29:39Z" message: InferenceGraph [splitter-graph-6d5c6] is Ready metadata: creationTimestamp: "2026-09-17T16:29:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:39Z" name: splitter-graph-6d5c6.18d628a23e5026f7 namespace: kserve-ci-e2e-test resourceVersion: "26259" uid: c557055e-a4c0-4b07-b4d2-04d3b00afddf reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:29:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: be79aad2-cb9c-498d-abf9-30104fadeb9f kind: Event lastTimestamp: "2026-09-17T16:30:24Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:30:24Z" name: splitter-graph-6d5c6.18d628a5b648a1bb namespace: kserve-ci-e2e-test resourceVersion: "26875" uid: 52d1b240-1b77-440c-ab36-3c048441475e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:29:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26238" uid: be79aad2-cb9c-498d-abf9-30104fadeb9f kind: Event lastTimestamp: "2026-09-17T16:30:24Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:29:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:30:24Z" name: splitter-graph-6d5c6.18d628a5b6f01083 namespace: kserve-ci-e2e-test resourceVersion: "26876" uid: 8dd6df6a-2360-4e87-b7ab-9c0c1248d473 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:28:28.744692Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25430" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-7ce9a-569dbd59-5z7h9 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:28:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:28:28Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62891d58ff154 namespace: kserve-ci-e2e-test resourceVersion: "25435" uid: 2c1acda7-cdbe-42da-b2a8-c3cbe8c6b4b7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:29Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25439" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:28:29Z" message: Add eth0 [10.132.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:28:29Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62891f7a82ab2 namespace: kserve-ci-e2e-test resourceVersion: "25453" uid: ea2cf13b-0386-40bf-a906-678e3c7bd1f4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7ce9a} kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:28:29Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:29Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62891f9232483 namespace: kserve-ci-e2e-test resourceVersion: "25455" uid: c6723f7d-fefd-43bb-8af0-335ea1d32d6d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7ce9a} kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:28:29Z" message: 'Created container: splitter-graph-7ce9a' metadata: creationTimestamp: "2026-09-17T16:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:29Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62891fc5a695e namespace: kserve-ci-e2e-test resourceVersion: "25456" uid: b156bfbd-ad0f-4396-b7e7-75a499d2c2b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7ce9a} kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:28:29Z" message: Started container splitter-graph-7ce9a metadata: creationTimestamp: "2026-09-17T16:28:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:29Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62891fce38783 namespace: kserve-ci-e2e-test resourceVersion: "25457" uid: 510a53c8-ec38-47b1-becd-3cb61bb395d3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7ce9a} kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:28:38Z" message: Stopping container splitter-graph-7ce9a metadata: creationTimestamp: "2026-09-17T16:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:38Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d628942c4ae724 namespace: kserve-ci-e2e-test resourceVersion: "25558" uid: cfc63a6d-09d6-409b-904f-3b3f30a38d70 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-7ce9a} kind: Pod name: splitter-graph-7ce9a-569dbd59-5z7h9 namespace: kserve-ci-e2e-test resourceVersion: "25431" uid: 92c33261-3945-41ca-9bc1-ed2c56ebc8f0 kind: Event lastTimestamp: "2026-09-17T16:29:06Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:28:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:06Z" name: splitter-graph-7ce9a-569dbd59-5z7h9.18d62894bb14e0ea namespace: kserve-ci-e2e-test resourceVersion: "25921" uid: e28abb53-0afa-4873-ac9a-e367e57390f3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-7ce9a-569dbd59 namespace: kserve-ci-e2e-test resourceVersion: "25427" uid: c6623806-d4ea-4511-a5eb-e316a7da7641 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: 'Created pod: splitter-graph-7ce9a-569dbd59-5z7h9' metadata: creationTimestamp: "2026-09-17T16:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: splitter-graph-7ce9a-569dbd59.18d62891d5119f32 namespace: kserve-ci-e2e-test resourceVersion: "25432" uid: f04a47bd-73dd-4682-ade4-cb78c6b8b104 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "25424" uid: 44a0b9fd-d89d-4d19-8309-32033759f00a kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: Scaled up replica set splitter-graph-7ce9a-569dbd59 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: splitter-graph-7ce9a.18d62891d3f6fe4a namespace: kserve-ci-e2e-test resourceVersion: "25428" uid: 74bf5748-fed6-4445-b6c8-26cdab3b3993 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:28Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "25443" uid: 3149accd-925a-4769-935d-cb22c973c7c3 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: InferenceGraph [splitter-graph-7ce9a] is Ready metadata: creationTimestamp: "2026-09-17T16:28:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:28Z" name: splitter-graph-7ce9a.18d62891dc5a9a86 namespace: kserve-ci-e2e-test resourceVersion: "25445" uid: 9a1a08f2-49bf-454d-b4b4-d6fdccadad15 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:07:48.154566Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15113" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d62770fca0ef90 namespace: kserve-ci-e2e-test resourceVersion: "15118" uid: 6937b5a2-1574-4796-aa1f-d5ad6cd4f5b1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15127" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627711ee9ca09 namespace: kserve-ci-e2e-test resourceVersion: "15162" uid: f98a2faf-e560-4e1d-adec-70412b5dbe9f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d6277120d9a0c5 namespace: kserve-ci-e2e-test resourceVersion: "15164" uid: cb69476d-c20f-469a-a5b5-3256a2452e66 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d6277123d108ad namespace: kserve-ci-e2e-test resourceVersion: "15165" uid: 8a4bd6a4-b174-4d37-a911-faf74dc7e57a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d62771247ad726 namespace: kserve-ci-e2e-test resourceVersion: "15166" uid: 1b456084-e1cb-4a25-962b-2137bfdbb856 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627712485e2f5 namespace: kserve-ci-e2e-test resourceVersion: "15168" uid: be794a30-ca18-4213-ae75-d8fd528c2a82 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d6277127bc89b8 namespace: kserve-ci-e2e-test resourceVersion: "15169" uid: c84ed67a-f4f3-44ac-a7b4-c039cc64d449 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d62771282cd8d9 namespace: kserve-ci-e2e-test resourceVersion: "15170" uid: 4c20ee61-19ca-4149-b2f4-64090779c9f8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:07:50Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:08:26Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:07:50Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:26Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d6277171201374 namespace: kserve-ci-e2e-test resourceVersion: "15532" uid: 28468c25-3769-456f-b60b-38be650c98dd reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:02Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627f227d9c3a8 namespace: kserve-ci-e2e-test resourceVersion: "19439" uid: db089b70-7a44-418c-ae4f-edfd9f4c636d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:02Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627f227da3908 namespace: kserve-ci-e2e-test resourceVersion: "19443" uid: 391815ea-9edf-4949-b4b2-165eb9100733 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:17:07Z" message: 'Readiness probe failed: Get "https://10.133.0.26:8643/healthz": net/http: request canceled while waiting for connection (Client.Timeout exceeded while awaiting headers)' metadata: creationTimestamp: "2026-09-17T16:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:07Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627f32151b303 namespace: kserve-ci-e2e-test resourceVersion: "19573" uid: c1cc1dd8-4c36-419a-99bf-12a10fc94566 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv namespace: kserve-ci-e2e-test resourceVersion: "15115" uid: e0bb7c72-2aa2-44bd-a8bd-72da01cf0a1d kind: Event lastTimestamp: "2026-09-17T16:17:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.26:8080: i/o timeout' metadata: creationTimestamp: "2026-09-17T16:17:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:07Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv.18d627f321a0307e namespace: kserve-ci-e2e-test resourceVersion: "19574" uid: 28a2af2e-6b5b-46ea-99eb-16b530ea7549 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0dabd-predictor-55d56cfcbb namespace: kserve-ci-e2e-test resourceVersion: "15106" uid: 53c8a4a9-2512-412c-803c-2bd100f4d394 kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: 'Created pod: success-200-isvc-0dabd-predictor-55d56cfcbb-j58wv' metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor-55d56cfcbb.18d62770fc078b86 namespace: kserve-ci-e2e-test resourceVersion: "15114" uid: 5e853771-8f02-4e78-af4a-e8c199104dc1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15104" uid: 7dc6cea8-b269-497f-8041-7aa3919f4b11 kind: Event lastTimestamp: "2026-09-17T16:07:48Z" message: Scaled up replica set success-200-isvc-0dabd-predictor-55d56cfcbb from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:48Z" name: success-200-isvc-0dabd-predictor.18d62770f82bee54 namespace: kserve-ci-e2e-test resourceVersion: "15107" uid: 2a055a37-83ae-467d-b13c-c7b3ce6cc46f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 9b42523a-629e-4b48-bd99-5cc7661fcb27 kind: Event lastTimestamp: "2026-09-17T16:08:18Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:18Z" name: success-200-isvc-0dabd-predictor.18d6277477fd7409 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 7e7f95bb-a26a-42c7-81eb-b17698da4545 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15105" uid: 9b42523a-629e-4b48-bd99-5cc7661fcb27 kind: Event lastTimestamp: "2026-09-17T16:08:18Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:18Z" name: success-200-isvc-0dabd-predictor.18d6277478a5f036 namespace: kserve-ci-e2e-test resourceVersion: "15464" uid: 1bd3e410-76e2-4b7c-9f7b-a060e78af267 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 9b42523a-629e-4b48-bd99-5cc7661fcb27 kind: Event lastTimestamp: "2026-09-17T16:09:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:03Z" name: success-200-isvc-0dabd-predictor.18d6277b77453f09 namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 84b9fc9f-d607-4534-8010-79485ee8fe95 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:08:33Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0dabd-predictor namespace: kserve-ci-e2e-test resourceVersion: "15314" uid: 9b42523a-629e-4b48-bd99-5cc7661fcb27 kind: Event lastTimestamp: "2026-09-17T16:09:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:33Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:03Z" name: success-200-isvc-0dabd-predictor.18d6277b781d16f2 namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 490ff6b2-1a74-4e77-a14f-7c7f62c41c23 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:07:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15094" uid: 74b8901d-bb14-4b8d-be4a-7959796eab19 kind: Event lastTimestamp: "2026-09-17T16:08:36Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:07:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:36Z" name: success-200-isvc-0dabd.18d62770f8346689 namespace: kserve-ci-e2e-test resourceVersion: "15788" uid: add80c58-6708-4428-a8d9-9711ba1628ff reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:36Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15785" uid: 74b8901d-bb14-4b8d-be4a-7959796eab19 kind: Event lastTimestamp: "2026-09-17T16:08:36Z" message: InferenceService [success-200-isvc-0dabd] is Ready metadata: creationTimestamp: "2026-09-17T16:08:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:36Z" name: success-200-isvc-0dabd.18d6277c3355b08f namespace: kserve-ci-e2e-test resourceVersion: "15787" uid: 32a5a767-57c2-4a03-89b5-d09784e309c2 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:13.575004Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20366" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d628029a9779d4 namespace: kserve-ci-e2e-test resourceVersion: "20373" uid: cf668a6f-f3db-4eba-b44b-8c22be9c929f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20376" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Add eth0 [10.133.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802bc68584a namespace: kserve-ci-e2e-test resourceVersion: "20420" uid: 677b2e5d-af8a-4d12-b773-f6e67743cc59 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802be8183fb namespace: kserve-ci-e2e-test resourceVersion: "20423" uid: da470188-8913-4420-97dc-1ced46c10152 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802c21278de namespace: kserve-ci-e2e-test resourceVersion: "20426" uid: d754665f-8451-47c5-bb84-76e8370fb433 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802c280ffb5 namespace: kserve-ci-e2e-test resourceVersion: "20427" uid: 20d6509a-b638-4b1f-b012-96826364245e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802c289b672 namespace: kserve-ci-e2e-test resourceVersion: "20428" uid: deb70354-fb0d-489d-a37d-3b4c049492a8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802c55d4635 namespace: kserve-ci-e2e-test resourceVersion: "20430" uid: c2d81ba6-13fc-4cad-b022-b1528a40cc43 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:14Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:14Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:14Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62802c5cc3e6d namespace: kserve-ci-e2e-test resourceVersion: "20431" uid: 374f6eaf-a731-4b14-ba54-a93f67ef55e9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:15Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:18:51Z" message: 'Readiness probe failed: dial tcp 10.133.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:51Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62803173975d2 namespace: kserve-ci-e2e-test resourceVersion: "20805" uid: d73302f1-ba40-47ef-af79-9d19edc77797 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62883d17f09ff namespace: kserve-ci-e2e-test resourceVersion: "24683" uid: 646fda3e-c9b2-480f-8e65-881a3b645680 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9 namespace: kserve-ci-e2e-test resourceVersion: "20370" uid: 9b8f143e-2616-4647-8996-2b5e7b4f03c1 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9.18d62883d17f85d6 namespace: kserve-ci-e2e-test resourceVersion: "24684" uid: d3aed93e-2167-4ae0-b9d9-ed1e097cd54a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-1e5cb-predictor-5d6b764d8 namespace: kserve-ci-e2e-test resourceVersion: "20359" uid: 0ccf3981-04e9-4862-8d1b-09a89b5b949a kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: 'Created pod: success-200-isvc-1e5cb-predictor-5d6b764d8-286f9' metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-1e5cb-predictor-5d6b764d8.18d628029968e656 namespace: kserve-ci-e2e-test resourceVersion: "20368" uid: e448c550-df19-4dc6-88e3-3240a1c17fe3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20357" uid: 0806b61c-2aba-48da-a8a0-736b2cf3c38a kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Scaled up replica set success-200-isvc-1e5cb-predictor-5d6b764d8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-1e5cb-predictor.18d6280296b5fe8c namespace: kserve-ci-e2e-test resourceVersion: "20361" uid: 090f5ca5-d065-4e96-bc20-5ccca0a11125 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: f037603a-6784-4541-82ab-b4a8176882dc kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: success-200-isvc-1e5cb-predictor.18d6280615575663 namespace: kserve-ci-e2e-test resourceVersion: "20710" uid: de00b18f-78a5-4c5c-8eae-1879ab276d4d reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20358" uid: f037603a-6784-4541-82ab-b4a8176882dc kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: success-200-isvc-1e5cb-predictor.18d6280615f80f52 namespace: kserve-ci-e2e-test resourceVersion: "20712" uid: f1afa802-b6eb-442a-8293-56b9d378b2e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:18:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f037603a-6784-4541-82ab-b4a8176882dc kind: Event lastTimestamp: "2026-09-17T16:19:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:28Z" name: success-200-isvc-1e5cb-predictor.18d6280d168b4aa8 namespace: kserve-ci-e2e-test resourceVersion: "21300" uid: f10c5a77-acf4-4823-96c9-ab75f49e1f59 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:18:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-1e5cb-predictor namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: f037603a-6784-4541-82ab-b4a8176882dc kind: Event lastTimestamp: "2026-09-17T16:19:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:28Z" name: success-200-isvc-1e5cb-predictor.18d6280d1715787b namespace: kserve-ci-e2e-test resourceVersion: "21302" uid: e7e4784c-fa4e-46c1-a0d6-8460d13ac725 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "20344" uid: 9795d650-d2ef-4e15-93e4-4deb46a79ed2 kind: Event lastTimestamp: "2026-09-17T16:19:01Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:01Z" name: success-200-isvc-1e5cb.18d6280296b605ec namespace: kserve-ci-e2e-test resourceVersion: "21047" uid: 814b77ff-bccd-41f1-bb68-8d0110b04da6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:01Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-1e5cb namespace: kserve-ci-e2e-test resourceVersion: "21041" uid: 9795d650-d2ef-4e15-93e4-4deb46a79ed2 kind: Event lastTimestamp: "2026-09-17T16:19:01Z" message: InferenceService [success-200-isvc-1e5cb] is Ready metadata: creationTimestamp: "2026-09-17T16:19:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:01Z" name: success-200-isvc-1e5cb.18d6280dd5176a43 namespace: kserve-ci-e2e-test resourceVersion: "21043" uid: 2f8b4f95-aad1-4c5c-8829-005f90a6fbcc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:18:53.972306Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20881" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c02755d6d namespace: kserve-ci-e2e-test resourceVersion: "20892" uid: d03528d6-2441-484c-8e66-21f9f74ac723 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20900" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Add eth0 [10.133.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c248cc077 namespace: kserve-ci-e2e-test resourceVersion: "20940" uid: 50be3655-667f-42b4-8d28-762606dbfbfb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c260fff58 namespace: kserve-ci-e2e-test resourceVersion: "20942" uid: d78f69cd-b098-4d2c-ba49-a79d8f2ff1b7 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c29089c74 namespace: kserve-ci-e2e-test resourceVersion: "20945" uid: 60afe29c-b361-4737-a8ea-15a3afceb416 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c2971c950 namespace: kserve-ci-e2e-test resourceVersion: "20946" uid: 52bad200-d67c-4835-813f-6bb3b77e8290 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c297a501d namespace: kserve-ci-e2e-test resourceVersion: "20947" uid: 87783679-4bed-4248-9e17-4f0266e12cb0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c2c615f2e namespace: kserve-ci-e2e-test resourceVersion: "20948" uid: ca8acd65-3fec-4c6e-83ab-bfe181bf48c1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:18:54Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:54Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c2cd2d5f5 namespace: kserve-ci-e2e-test resourceVersion: "20949" uid: c704ef47-57b5-4d42-b7f6-fa95c2469e78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:18:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:19:31Z" message: 'Readiness probe failed: dial tcp 10.133.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:19:31Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6280c7118c962 namespace: kserve-ci-e2e-test resourceVersion: "21330" uid: aaca57bf-7ad9-4c60-a14f-8e77b77ca227 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6288d291ac187 namespace: kserve-ci-e2e-test resourceVersion: "25135" uid: 3f7e4746-38c0-4a83-b02c-719e7c6540ff reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt namespace: kserve-ci-e2e-test resourceVersion: "20884" uid: 5c6c613c-70f1-4931-a919-984d068f821a kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt.18d6288d291b60f6 namespace: kserve-ci-e2e-test resourceVersion: "25139" uid: 2ab1db1d-c4db-4c3d-ac29-5e2c5b1648e0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-26f37-predictor-7c64bddcdd namespace: kserve-ci-e2e-test resourceVersion: "20865" uid: ea05c19b-af2c-42a9-99fa-7248bdd0bfd0 kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: 'Created pod: success-200-isvc-26f37-predictor-7c64bddcdd-cg5kt' metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-26f37-predictor-7c64bddcdd.18d6280c01a8f2a6 namespace: kserve-ci-e2e-test resourceVersion: "20888" uid: 1cde09ce-8990-47ff-be98-b9e01c2cdfbe reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20862" uid: 15bcb103-0a38-4b74-a7e2-d3c7bcd57398 kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Scaled up replica set success-200-isvc-26f37-predictor-7c64bddcdd from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-26f37-predictor.18d6280bfc447c70 namespace: kserve-ci-e2e-test resourceVersion: "20867" uid: be1eecac-8e60-4d3b-9440-426acfc91a1b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 3b3504ef-18c0-4029-b130-eef4216c1614 kind: Event lastTimestamp: "2026-09-17T16:19:08Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:08Z" name: success-200-isvc-26f37-predictor.18d6280f7bb29b1f namespace: kserve-ci-e2e-test resourceVersion: "21101" uid: 646b4f83-d66a-4021-8b1d-ac3ad3556fef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:08Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "20864" uid: 3b3504ef-18c0-4029-b130-eef4216c1614 kind: Event lastTimestamp: "2026-09-17T16:19:08Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:19:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:19:08Z" name: success-200-isvc-26f37-predictor.18d6280f7c1085ac namespace: kserve-ci-e2e-test resourceVersion: "21102" uid: 9a96ed79-db67-4e29-a22e-e248039c0a24 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:19:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 3b3504ef-18c0-4029-b130-eef4216c1614 kind: Event lastTimestamp: "2026-09-17T16:20:09Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:09Z" name: success-200-isvc-26f37-predictor.18d62812fb30c24c namespace: kserve-ci-e2e-test resourceVersion: "21652" uid: bba8be23-8977-4824-8080-bfc050bcdcc7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:19:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-26f37-predictor namespace: kserve-ci-e2e-test resourceVersion: "21100" uid: 3b3504ef-18c0-4029-b130-eef4216c1614 kind: Event lastTimestamp: "2026-09-17T16:20:09Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:19:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:20:09Z" name: success-200-isvc-26f37-predictor.18d62812fc08db6f namespace: kserve-ci-e2e-test resourceVersion: "21654" uid: 2d128e08-34d2-4ada-80b2-775f7e20802a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 6c22a1dc-2ab8-4c34-ad91-e1a2def573d4 kind: Event lastTimestamp: "2026-09-17T16:19:41Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:41Z" name: success-200-isvc-26f37.18d6280bfc5f4060 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: 285acc73-1496-4d7d-a77e-1cc95bdf7bbc reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 6c22a1dc-2ab8-4c34-ad91-e1a2def573d4 kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: 'Failed to update status for InferenceService "success-200-isvc-26f37": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-26f37.18d6280bfd9a27c7 namespace: kserve-ci-e2e-test resourceVersion: "20870" uid: f1ad7964-2255-44b5-98a1-e6169c2c0a62 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: 6c22a1dc-2ab8-4c34-ad91-e1a2def573d4 kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-26f37": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-26f37.18d6280bfd9a8d57 namespace: kserve-ci-e2e-test resourceVersion: "20874" uid: 3a306491-8d92-4239-b936-2645dff14797 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:19:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-26f37 namespace: kserve-ci-e2e-test resourceVersion: "21406" uid: 6c22a1dc-2ab8-4c34-ad91-e1a2def573d4 kind: Event lastTimestamp: "2026-09-17T16:19:41Z" message: InferenceService [success-200-isvc-26f37] is Ready metadata: creationTimestamp: "2026-09-17T16:19:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:19:41Z" name: success-200-isvc-26f37.18d628172e0a4bde namespace: kserve-ci-e2e-test resourceVersion: "21408" uid: f769f89e-f904-4f5c-ba81-5c93be9055c3 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:28:08.771796Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25163" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d2f15b7ff namespace: kserve-ci-e2e-test resourceVersion: "25168" uid: 3d1dc077-4fc7-4e2b-a5da-3a5db06f519f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25173" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Add eth0 [10.133.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d50da6e42 namespace: kserve-ci-e2e-test resourceVersion: "25213" uid: c889043b-e773-4dcb-854e-c63c30c98b06 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d526fe608 namespace: kserve-ci-e2e-test resourceVersion: "25215" uid: bb9199b1-6bd8-429d-be6b-52a862b3f312 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d5550ffb1 namespace: kserve-ci-e2e-test resourceVersion: "25216" uid: 1c4cff77-6009-4d3a-8d32-a777b4c84170 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d55b9db5f namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 9f9212c5-5bd2-49bd-be3c-08c67a23379b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d55c2e766 namespace: kserve-ci-e2e-test resourceVersion: "25219" uid: 5b6148c1-faf4-48e1-b518-e1ce7c7cede4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d58e22289 namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 44d788b0-368e-4229-8cdb-42fbe595d972 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:09Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:09Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d594ff040 namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 6197b291-be46-4895-8ced-cc2f0ea88fab reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:28:46Z" message: 'Readiness probe failed: dial tcp 10.133.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:28:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:46Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6288d7a0ae878 namespace: kserve-ci-e2e-test resourceVersion: "25740" uid: 2820c2f5-afed-4c27-979c-b1a47d1502e5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:45:28Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:28Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6297f3cf57cc6 namespace: kserve-ci-e2e-test resourceVersion: "33131" uid: 4e039ed6-3907-43c0-bff8-9d79c03ea0f8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:45:28Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:28Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6297f3cf664af namespace: kserve-ci-e2e-test resourceVersion: "33133" uid: 0b493789-65f4-460d-99b1-13a51ecf4767 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk namespace: kserve-ci-e2e-test resourceVersion: "25165" uid: a0cf25ad-3c17-44f1-b5ae-b2a0b621d1ef kind: Event lastTimestamp: "2026-09-17T16:45:31Z" message: 'Readiness probe failed: Get "https://10.133.0.33:8643/healthz": dial tcp 10.133.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:31Z" name: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk.18d6297fdaa1c979 namespace: kserve-ci-e2e-test resourceVersion: "33200" uid: e45c75cc-d279-488a-a2a0-aa6712285a10 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-37c70-predictor-6b44d5b64d namespace: kserve-ci-e2e-test resourceVersion: "25152" uid: 649821f6-c8fb-4135-8823-8eb391b802bd kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'Created pod: success-200-isvc-37c70-predictor-6b44d5b64d-bp6hk' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-37c70-predictor-6b44d5b64d.18d6288d2e197836 namespace: kserve-ci-e2e-test resourceVersion: "25164" uid: 3c6a5820-b93b-4b80-85df-36084b888446 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25148" uid: ea6533cd-6237-4261-8a87-004b6f0ff97a kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: Scaled up replica set success-200-isvc-37c70-predictor-6b44d5b64d from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-37c70-predictor.18d6288d2b67e7bb namespace: kserve-ci-e2e-test resourceVersion: "25153" uid: efc39f6f-10f5-4690-886b-74c9e0725d08 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25149" uid: a36447fd-f15d-4f83-b4f4-79b84493802f kind: Event lastTimestamp: "2026-09-17T16:28:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:38Z" name: success-200-isvc-37c70-predictor.18d62890aa13bb29 namespace: kserve-ci-e2e-test resourceVersion: "25544" uid: e241c3d4-282b-49e9-81bf-0c09dcab7398 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:28:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25149" uid: a36447fd-f15d-4f83-b4f4-79b84493802f kind: Event lastTimestamp: "2026-09-17T16:28:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:38Z" name: success-200-isvc-37c70-predictor.18d62890aa9954c0 namespace: kserve-ci-e2e-test resourceVersion: "25550" uid: b4bafdb4-1e15-41ac-a286-fdd82bbb6cfd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: a36447fd-f15d-4f83-b4f4-79b84493802f kind: Event lastTimestamp: "2026-09-17T16:29:23Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:23Z" name: success-200-isvc-37c70-predictor.18d62897a94ab09d namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: d07d20b5-17e3-4301-b43f-394a4157d90a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:28:53Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-37c70-predictor namespace: kserve-ci-e2e-test resourceVersion: "25373" uid: a36447fd-f15d-4f83-b4f4-79b84493802f kind: Event lastTimestamp: "2026-09-17T16:29:23Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:28:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:23Z" name: success-200-isvc-37c70-predictor.18d62897a9f85ea9 namespace: kserve-ci-e2e-test resourceVersion: "26094" uid: cef2f25f-bbe7-459f-b783-783e4077f902 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: c37f6c97-dceb-4d96-8b30-ccb7d76854f6 kind: Event lastTimestamp: "2026-09-17T16:28:56Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:56Z" name: success-200-isvc-37c70.18d6288d2b614c24 namespace: kserve-ci-e2e-test resourceVersion: "25840" uid: b541650f-4844-4259-9e1f-391269f8ee3b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: c37f6c97-dceb-4d96-8b30-ccb7d76854f6 kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'Failed to update status for InferenceService "success-200-isvc-37c70": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-37c70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-37c70.18d6288d2c05eca4 namespace: kserve-ci-e2e-test resourceVersion: "25158" uid: 94e06d8b-3361-4774-b513-3d8062473dbe reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:08Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25116" uid: c37f6c97-dceb-4d96-8b30-ccb7d76854f6 kind: Event lastTimestamp: "2026-09-17T16:28:08Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-37c70": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:28:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:08Z" name: success-200-isvc-37c70.18d6288d2c06400a namespace: kserve-ci-e2e-test resourceVersion: "25159" uid: 8adea113-42a9-4220-af28-ef36b65d5e9a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25834" uid: c37f6c97-dceb-4d96-8b30-ccb7d76854f6 kind: Event lastTimestamp: "2026-09-17T16:28:56Z" message: InferenceService [success-200-isvc-37c70] is Ready metadata: creationTimestamp: "2026-09-17T16:28:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:56Z" name: success-200-isvc-37c70.18d62898376997a5 namespace: kserve-ci-e2e-test resourceVersion: "25836" uid: 6e0a2da3-d625-4eb0-9994-bfcafb22a482 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:17:43.622298Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19925" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-48574-predictor-69dd69f964-fv7vm to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fba1456a8b namespace: kserve-ci-e2e-test resourceVersion: "19930" uid: e68b3a0e-753c-4bb6-b057-dbf4af9d39fc reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-48574-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbad006b94 namespace: kserve-ci-e2e-test resourceVersion: "19962" uid: 66ccd669-f347-4d05-aa65-c961d078c52a reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19935" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbe7309cae namespace: kserve-ci-e2e-test resourceVersion: "19977" uid: 779ac37a-0814-48c2-950c-efd577f6b0bc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbe8b3ffbb namespace: kserve-ci-e2e-test resourceVersion: "19980" uid: fb9fa4aa-4ff8-42a0-a0b7-fb0c6c2a0d83 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbec112ac1 namespace: kserve-ci-e2e-test resourceVersion: "19981" uid: cfab3a6f-d611-474e-9f9c-66db1384fbdb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbec8c2e01 namespace: kserve-ci-e2e-test resourceVersion: "19982" uid: 80c3244c-70c4-4de4-95f0-44e9d165f3e0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbec979507 namespace: kserve-ci-e2e-test resourceVersion: "19983" uid: 21bf273b-5f02-43f6-850c-c39c8d68218e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbefccc916 namespace: kserve-ci-e2e-test resourceVersion: "19987" uid: 331d461c-1ca6-4189-95a3-d56373c9114c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:17:44Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:44Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fbf05c21ca namespace: kserve-ci-e2e-test resourceVersion: "19988" uid: c3cdd653-eaa7-4a0c-9960-d2c68ce93653 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:18:21Z" message: 'Readiness probe failed: dial tcp 10.133.0.29:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:17:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:21Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d627fc12d0aefc namespace: kserve-ci-e2e-test resourceVersion: "20507" uid: ce17a7fa-16e0-45e9-8344-ace1fb4c80ad reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d6280bf9b70233 namespace: kserve-ci-e2e-test resourceVersion: "20858" uid: 5254bc7d-a288-42e9-8fe9-133f7036edf4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:18:53Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:53Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d6280bf9b77681 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 6b57657b-53f8-41c2-89e8-76e6b0f586df reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48574-predictor-69dd69f964-fv7vm namespace: kserve-ci-e2e-test resourceVersion: "19927" uid: 5f0ccf71-53ce-4105-a9bc-67974be20b8d kind: Event lastTimestamp: "2026-09-17T16:18:56Z" message: 'Readiness probe failed: Get "https://10.133.0.29:8643/healthz": dial tcp 10.133.0.29:8643: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:18:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:56Z" name: success-200-isvc-48574-predictor-69dd69f964-fv7vm.18d6280c9af16f28 namespace: kserve-ci-e2e-test resourceVersion: "20976" uid: a8b0232b-5b8d-4155-931e-c7ea5c5cbf42 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-48574-predictor-69dd69f964 namespace: kserve-ci-e2e-test resourceVersion: "19919" uid: 78d1071a-9bb5-45b5-b07c-93e1188ab69b kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: 'Created pod: success-200-isvc-48574-predictor-69dd69f964-fv7vm' metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-48574-predictor-69dd69f964.18d627fba0bed1db namespace: kserve-ci-e2e-test resourceVersion: "19926" uid: f23d7bab-bb41-46a1-8d94-f87cc2cf4f63 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19917" uid: af71f378-4d4f-456f-b59f-d472577eb145 kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Scaled up replica set success-200-isvc-48574-predictor-69dd69f964 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-48574-predictor.18d627fb9e841eab namespace: kserve-ci-e2e-test resourceVersion: "19920" uid: 96553452-56a1-43a9-9f7f-7c6ede7e5c68 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:17:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 5afd4aa3-fc03-4b0e-b286-ff897380447c kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-48574-predictor.18d627ff1d5583e2 namespace: kserve-ci-e2e-test resourceVersion: "20377" uid: 5a6e5af8-cbcc-47af-bc0e-caa6b3652304 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:17:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "19918" uid: 5afd4aa3-fc03-4b0e-b286-ff897380447c kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-48574-predictor.18d627ff1e032694 namespace: kserve-ci-e2e-test resourceVersion: "20381" uid: e7797f76-9fbc-492e-bcaa-b2fdaa0afc4b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 5afd4aa3-fc03-4b0e-b286-ff897380447c kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: success-200-isvc-48574-predictor.18d628061c035c7e namespace: kserve-ci-e2e-test resourceVersion: "20720" uid: 1376fbdd-54cc-466a-8d67-f64cb3669443 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:18:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48574-predictor namespace: kserve-ci-e2e-test resourceVersion: "20134" uid: 5afd4aa3-fc03-4b0e-b286-ff897380447c kind: Event lastTimestamp: "2026-09-17T16:18:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:18:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:43Z" name: success-200-isvc-48574-predictor.18d628061ce1c8d2 namespace: kserve-ci-e2e-test resourceVersion: "20724" uid: 35a2d3da-dd92-4291-a5b4-9cd3e2152c0c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48574 namespace: kserve-ci-e2e-test resourceVersion: "19907" uid: 952ae330-afc4-4b4e-a570-2ffb967dc138 kind: Event lastTimestamp: "2026-09-17T16:18:31Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:31Z" name: success-200-isvc-48574.18d627fb9eaa3e51 namespace: kserve-ci-e2e-test resourceVersion: "20612" uid: a539945b-5e7b-4fd0-a8a1-899183586f30 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:31Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48574 namespace: kserve-ci-e2e-test resourceVersion: "20609" uid: 952ae330-afc4-4b4e-a570-2ffb967dc138 kind: Event lastTimestamp: "2026-09-17T16:18:31Z" message: InferenceService [success-200-isvc-48574] is Ready metadata: creationTimestamp: "2026-09-17T16:18:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:18:31Z" name: success-200-isvc-48574.18d62806d2adb693 namespace: kserve-ci-e2e-test resourceVersion: "20610" uid: 0fdad5a0-881b-474b-b3da-0fc091b8d704 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:05:58.017209Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13892" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:05:58Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275757ee461c namespace: kserve-ci-e2e-test resourceVersion: "13898" uid: ce869c66-2023-4d5c-a2c4-9afd57c613bd reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13908" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:05:58Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d62757799728d5 namespace: kserve-ci-e2e-test resourceVersion: "13972" uid: 98daf98d-f731-44f7-94a3-5f9e62a16d3e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:05:58Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d627577bcb3f3d namespace: kserve-ci-e2e-test resourceVersion: "13978" uid: 2c6ccad9-6c2e-44c4-8a15-f2007ff16fc0 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" in 27.67s (27.67s including waiting). Image size: 1356848206 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275ded1b6a98 namespace: kserve-ci-e2e-test resourceVersion: "14278" uid: 5fa9d817-f3d0-4092-80eb-4a4470bdcf0d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275dfa7e4abc namespace: kserve-ci-e2e-test resourceVersion: "14282" uid: c07a73cd-20f3-436c-8d64-6ccced1b39b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275dfc07ad93 namespace: kserve-ci-e2e-test resourceVersion: "14283" uid: 92838a20-f925-4056-8ae0-63ff800df8ed reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:26Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-09-17T16:06:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:26Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275dfc153a65 namespace: kserve-ci-e2e-test resourceVersion: "14284" uid: a8aa8ab5-bb6c-444a-8e86-1c889c6cba1e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 3.305s (3.305s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275ec11afdf7 namespace: kserve-ci-e2e-test resourceVersion: "14329" uid: 08bd7ac3-7d52-48a6-b0d4-af446f681c9e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275ec4383b21 namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 8b40b4da-bd0b-48e3-82d0-6019260149d9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:06:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:06:29Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:06:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:06:29Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275ec49e3818 namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 3fc8dd86-2704-49be-9844-29ffb38cadf7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:06:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:07:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.23:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:06:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:07Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d6275f33862fda namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: f2a6eaf4-5fbb-4c69-a292-0f6cd804047d reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:07:47Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:47Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d62770f0e46edd namespace: kserve-ci-e2e-test resourceVersion: "15081" uid: c2f99155-1eda-42b8-9aa3-072bc03ea93c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4 namespace: kserve-ci-e2e-test resourceVersion: "13895" uid: 17e23660-3718-47a9-bdde-bdf34dd8c4c0 kind: Event lastTimestamp: "2026-09-17T16:07:47Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:47Z" name: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4.18d62770f0e4ed46 namespace: kserve-ci-e2e-test resourceVersion: "15086" uid: 75313212-6f2a-4953-a3bf-d954abf57b55 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:58Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-48b6f-predictor-7bcc757b48 namespace: kserve-ci-e2e-test resourceVersion: "13885" uid: 04708af8-ae9f-431e-9e45-e5fe55b5cb81 kind: Event lastTimestamp: "2026-09-17T16:05:58Z" message: 'Created pod: success-200-isvc-48b6f-predictor-7bcc757b48-6r5h4' metadata: creationTimestamp: "2026-09-17T16:05:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:58Z" name: success-200-isvc-48b6f-predictor-7bcc757b48.18d6275757628787 namespace: kserve-ci-e2e-test resourceVersion: "13894" uid: c96f54c7-1562-4d5f-937f-ac8a0214f6ae reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:05:57Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13882" uid: 074f3c43-4628-4f1d-a37c-f1aebccedb96 kind: Event lastTimestamp: "2026-09-17T16:05:57Z" message: Scaled up replica set success-200-isvc-48b6f-predictor-7bcc757b48 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:05:57Z" name: success-200-isvc-48b6f-predictor.18d6275751488f82 namespace: kserve-ci-e2e-test resourceVersion: "13887" uid: c90fb994-770f-4679-9473-8d5230ce9ed0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 66328ee0-fca5-4ab7-8438-fb0d46ac42d0 kind: Event lastTimestamp: "2026-09-17T16:07:13Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:13Z" name: success-200-isvc-48b6f-predictor.18d6275ad0ab5b0c namespace: kserve-ci-e2e-test resourceVersion: "14700" uid: e8fde5b6-fb4f-4dac-98f6-35024852d9a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:06:12Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "13884" uid: 66328ee0-fca5-4ab7-8438-fb0d46ac42d0 kind: Event lastTimestamp: "2026-09-17T16:07:13Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:06:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:13Z" name: success-200-isvc-48b6f-predictor.18d6275ad12ef0d0 namespace: kserve-ci-e2e-test resourceVersion: "14701" uid: 22d95c23-c259-431b-9e91-2dc8847a7e72 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 66328ee0-fca5-4ab7-8438-fb0d46ac42d0 kind: Event lastTimestamp: "2026-09-17T16:07:43Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:43Z" name: success-200-isvc-48b6f-predictor.18d6276c4c8a164f namespace: kserve-ci-e2e-test resourceVersion: "14987" uid: 9e027f9f-be77-4680-8ce3-f1478fbafa36 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:07:28Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-48b6f-predictor namespace: kserve-ci-e2e-test resourceVersion: "14163" uid: 66328ee0-fca5-4ab7-8438-fb0d46ac42d0 kind: Event lastTimestamp: "2026-09-17T16:07:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:07:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:43Z" name: success-200-isvc-48b6f-predictor.18d6276c4d25cab5 namespace: kserve-ci-e2e-test resourceVersion: "14988" uid: 005f498b-f436-4de8-875f-4ba94e113d73 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:05:57Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48b6f namespace: kserve-ci-e2e-test resourceVersion: "13862" uid: 35bf065f-3a80-4f9b-aa31-b86acd29891f kind: Event lastTimestamp: "2026-09-17T16:07:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:05:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:17Z" name: success-200-isvc-48b6f.18d62757520c55f5 namespace: kserve-ci-e2e-test resourceVersion: "14751" uid: 69a48e7d-3176-48e6-a39f-75f2368ddc07 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-48b6f namespace: kserve-ci-e2e-test resourceVersion: "14749" uid: 35bf065f-3a80-4f9b-aa31-b86acd29891f kind: Event lastTimestamp: "2026-09-17T16:07:17Z" message: InferenceService [success-200-isvc-48b6f] is Ready metadata: creationTimestamp: "2026-09-17T16:07:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:17Z" name: success-200-isvc-48b6f.18d62769f04330ce namespace: kserve-ci-e2e-test resourceVersion: "14750" uid: 07781e75-c8eb-4abb-bc01-9593a89f3382 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:08:29.016366Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15646" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:08:29Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:08:29Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277a802e8a69 namespace: kserve-ci-e2e-test resourceVersion: "15654" uid: 47deffcf-a1c3-4840-93ca-8c5db195fc0c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:30Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:30Z" message: 'MountVolume.SetUp failed for volume "kube-rbac-proxy-sar-config" : failed to sync configmap cache: timed out waiting for the condition' metadata: creationTimestamp: "2026-09-17T16:08:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:30Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277acc780116 namespace: kserve-ci-e2e-test resourceVersion: "15705" uid: 05062950-00a2-4b07-bd61-d212fabc7314 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15655" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277afb960cdf namespace: kserve-ci-e2e-test resourceVersion: "15717" uid: c1a15d76-ac9a-48c9-842d-7cf53769079f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277afdbc38ba namespace: kserve-ci-e2e-test resourceVersion: "15719" uid: 3e385ea7-11dc-430c-b9aa-21b20328d389 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b00e5e7b3 namespace: kserve-ci-e2e-test resourceVersion: "15720" uid: 56be8d25-590b-4889-9ccc-58388760f7c3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b014bf263 namespace: kserve-ci-e2e-test resourceVersion: "15721" uid: ffb4a6ed-c613-403e-bfd7-eeda5b254c3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b0156cdd1 namespace: kserve-ci-e2e-test resourceVersion: "15722" uid: c6166efa-27b0-4ad4-9371-89b9dc0657e6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b03e7444e namespace: kserve-ci-e2e-test resourceVersion: "15723" uid: c7bc36fe-990c-4a4f-8437-76a546abb095 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:08:31Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:31Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b04486f6b namespace: kserve-ci-e2e-test resourceVersion: "15724" uid: e89012f9-8e07-4e65-a04f-da3e7718e206 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:08:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:09:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.27:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:08:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:09:07Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d6277b07cb060e namespace: kserve-ci-e2e-test resourceVersion: "16088" uid: 5bf79e77-1d3a-4ac4-bcca-24763f3943ba reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d627fb980750af namespace: kserve-ci-e2e-test resourceVersion: "19893" uid: 1126ed0c-895c-49e1-97b2-81f652fc093a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v namespace: kserve-ci-e2e-test resourceVersion: "15648" uid: 4db8d94f-aacf-43d1-aeb3-4760ccb0ad90 kind: Event lastTimestamp: "2026-09-17T16:17:43Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:43Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v.18d627fb9807da3c namespace: kserve-ci-e2e-test resourceVersion: "19894" uid: af8b0361-75ad-4d8c-8944-47a81b24d380 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-4f67a-predictor-86bc4cfbd8 namespace: kserve-ci-e2e-test resourceVersion: "15640" uid: f457867e-b414-4672-a76c-5b6fa4dfbe38 kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: 'Created pod: success-200-isvc-4f67a-predictor-86bc4cfbd8-kp27v' metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:28Z" name: success-200-isvc-4f67a-predictor-86bc4cfbd8.18d6277a7eca88bb namespace: kserve-ci-e2e-test resourceVersion: "15647" uid: 5446e8aa-6607-4222-9cde-c3c546940192 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15638" uid: 42600dbc-a684-4066-bcfa-e70164ea2e6d kind: Event lastTimestamp: "2026-09-17T16:08:28Z" message: Scaled up replica set success-200-isvc-4f67a-predictor-86bc4cfbd8 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:28Z" name: success-200-isvc-4f67a-predictor.18d6277a7ad6d937 namespace: kserve-ci-e2e-test resourceVersion: "15641" uid: 424c518b-349a-4791-ac1c-2e9f5e465e1f reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: df1a7b83-57c8-4ec0-96eb-3dc07096936c kind: Event lastTimestamp: "2026-09-17T16:09:14Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:14Z" name: success-200-isvc-4f67a-predictor.18d6277dfa7558f6 namespace: kserve-ci-e2e-test resourceVersion: "16138" uid: 8454ff79-72af-4f45-a6aa-d690e0290486 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:08:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15639" uid: df1a7b83-57c8-4ec0-96eb-3dc07096936c kind: Event lastTimestamp: "2026-09-17T16:09:14Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:08:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:14Z" name: success-200-isvc-4f67a-predictor.18d6277dfb6ac3ae namespace: kserve-ci-e2e-test resourceVersion: "16140" uid: 93bcb077-dbff-45bd-8a0d-79cd7df4a8ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:08:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: df1a7b83-57c8-4ec0-96eb-3dc07096936c kind: Event lastTimestamp: "2026-09-17T16:09:44Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:44Z" name: success-200-isvc-4f67a-predictor.18d627817ac74521 namespace: kserve-ci-e2e-test resourceVersion: "16412" uid: e0528c22-34ea-4fb4-a376-37e22e4ad0d5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:08:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-4f67a-predictor namespace: kserve-ci-e2e-test resourceVersion: "15861" uid: df1a7b83-57c8-4ec0-96eb-3dc07096936c kind: Event lastTimestamp: "2026-09-17T16:09:44Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:08:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:44Z" name: success-200-isvc-4f67a-predictor.18d627817b50dd96 namespace: kserve-ci-e2e-test resourceVersion: "16413" uid: 2a1bcb88-1440-4501-8b53-66dbe3a75c39 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:08:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f67a namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 4266f7fa-66c7-446c-9caa-e399adb34279 kind: Event lastTimestamp: "2026-09-17T16:09:17Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:08:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:09:17Z" name: success-200-isvc-4f67a.18d6277a7bff7138 namespace: kserve-ci-e2e-test resourceVersion: "16181" uid: 1bf62727-4e77-48da-9b54-7cd1b05539f5 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:17Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-4f67a namespace: kserve-ci-e2e-test resourceVersion: "16177" uid: 4266f7fa-66c7-446c-9caa-e399adb34279 kind: Event lastTimestamp: "2026-09-17T16:09:17Z" message: InferenceService [success-200-isvc-4f67a] is Ready metadata: creationTimestamp: "2026-09-17T16:09:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:09:17Z" name: success-200-isvc-4f67a.18d62785c5828644 namespace: kserve-ci-e2e-test resourceVersion: "16180" uid: e1d152e0-3e8d-4b9a-80a9-851ad78c739e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:28:39.147160Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25615" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:28:39Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d62894419912ff namespace: kserve-ci-e2e-test resourceVersion: "25620" uid: ce0998dc-79d9-40cb-b1ca-e3dbd18fba42 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-6d5c6-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:39Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628944fc6ea9f namespace: kserve-ci-e2e-test resourceVersion: "25654" uid: d7f6b68a-c823-49bc-a5de-e625c733d9e7 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25625" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Add eth0 [10.133.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d62894872bea9e namespace: kserve-ci-e2e-test resourceVersion: "25668" uid: bfdc6f2f-8751-4554-977a-d47ef19eda56 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d62894889e9d7f namespace: kserve-ci-e2e-test resourceVersion: "25671" uid: 02754b30-3873-43b7-9113-1f1583192572 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628948baa0034 namespace: kserve-ci-e2e-test resourceVersion: "25672" uid: 3ad1b263-04ae-4af6-959f-8f16a3df73fb reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628948c192dfe namespace: kserve-ci-e2e-test resourceVersion: "25673" uid: b48524a6-d624-43b6-b652-7ad37d3b9f3b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628948c236b15 namespace: kserve-ci-e2e-test resourceVersion: "25674" uid: 728c21a4-5735-478d-9b06-9b74d0ec8547 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628948ef48fc8 namespace: kserve-ci-e2e-test resourceVersion: "25676" uid: 6e6f4848-9244-40ea-8b8c-d9bb016f44e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:28:40Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:40Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d628948f6926b1 namespace: kserve-ci-e2e-test resourceVersion: "25677" uid: a96c227b-c11d-4266-9546-4be964a464eb reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:29:18Z" message: 'Readiness probe failed: dial tcp 10.133.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:28:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:18Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d62894f56b5d68 namespace: kserve-ci-e2e-test resourceVersion: "26057" uid: d050ba63-c0b6-4c51-9634-acfc0c8b62b7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:37:54Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:37:54Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d6291574c43f98 namespace: kserve-ci-e2e-test resourceVersion: "29999" uid: 75637af6-a5cf-4062-ba8d-00c88c74ad89 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:37:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv namespace: kserve-ci-e2e-test resourceVersion: "25617" uid: bff3aff9-526f-4fe0-aaf7-a45a1057942e kind: Event lastTimestamp: "2026-09-17T16:37:54Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:37:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:37:54Z" name: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv.18d6291574c4b985 namespace: kserve-ci-e2e-test resourceVersion: "30002" uid: c98a94a7-f821-48bf-9a34-8d2d95ea047b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-6d5c6-predictor-b6584cc56 namespace: kserve-ci-e2e-test resourceVersion: "25609" uid: 5083335f-96e6-49e2-b49d-2ac642bc5b24 kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: 'Created pod: success-200-isvc-6d5c6-predictor-b6584cc56-qrgxv' metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:39Z" name: success-200-isvc-6d5c6-predictor-b6584cc56.18d628944121249e namespace: kserve-ci-e2e-test resourceVersion: "25616" uid: 7aa2b789-ec15-4215-a08a-60fedb2a865e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25606" uid: 9e0c95d3-5c77-46df-b02d-dc02ae582c9a kind: Event lastTimestamp: "2026-09-17T16:28:39Z" message: Scaled up replica set success-200-isvc-6d5c6-predictor-b6584cc56 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:39Z" name: success-200-isvc-6d5c6-predictor.18d628943e5357d1 namespace: kserve-ci-e2e-test resourceVersion: "25611" uid: b11cb835-888d-4a12-8557-3a66b43c8210 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25608" uid: 8c8008ba-302f-4a96-b429-445a407d0294 kind: Event lastTimestamp: "2026-09-17T16:28:54Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:54Z" name: success-200-isvc-6d5c6-predictor.18d62897bcd0b4a1 namespace: kserve-ci-e2e-test resourceVersion: "25812" uid: 562e3813-5960-482f-be0f-ea85448c2018 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:54Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25608" uid: 8c8008ba-302f-4a96-b429-445a407d0294 kind: Event lastTimestamp: "2026-09-17T16:28:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:28:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:54Z" name: success-200-isvc-6d5c6-predictor.18d62897bd4afe25 namespace: kserve-ci-e2e-test resourceVersion: "25813" uid: 12879cdd-858c-4bc9-92cc-a3263e51fbb2 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25811" uid: 8c8008ba-302f-4a96-b429-445a407d0294 kind: Event lastTimestamp: "2026-09-17T16:29:54Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:54Z" name: success-200-isvc-6d5c6-predictor.18d6289b3d0aafef namespace: kserve-ci-e2e-test resourceVersion: "26379" uid: a1141769-391d-4761-b59b-f7c84f77dc37 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-6d5c6-predictor namespace: kserve-ci-e2e-test resourceVersion: "25811" uid: 8c8008ba-302f-4a96-b429-445a407d0294 kind: Event lastTimestamp: "2026-09-17T16:29:54Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:54Z" name: success-200-isvc-6d5c6-predictor.18d6289b3e233b22 namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: c12b2c31-5c40-447d-ae4a-bdd832d19723 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:28:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "25592" uid: 5be4f1d8-25db-4359-b287-1af6dc9bc3df kind: Event lastTimestamp: "2026-09-17T16:29:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:28:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:28Z" name: success-200-isvc-6d5c6.18d628943e0631a4 namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 11895156-e58a-4075-ac1c-a8fbfeb971e8 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-6d5c6 namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 5be4f1d8-25db-4359-b287-1af6dc9bc3df kind: Event lastTimestamp: "2026-09-17T16:29:28Z" message: InferenceService [success-200-isvc-6d5c6] is Ready metadata: creationTimestamp: "2026-09-17T16:29:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:28Z" name: success-200-isvc-6d5c6.18d6289fb19914af namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: f53c3eb3-6c62-4ab0-a190-5a97009baacc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:17:03.052073Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19456" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:17:03Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f22f18f2bd namespace: kserve-ci-e2e-test resourceVersion: "19470" uid: 9788c57f-12f5-4f6f-9594-225d74284d5c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7c0d2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:03Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f23c6c17f7 namespace: kserve-ci-e2e-test resourceVersion: "19508" uid: bfee7839-2dc8-404e-98de-6aab8bd1835c reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19482" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f274bb2fcc namespace: kserve-ci-e2e-test resourceVersion: "19524" uid: eaecdd5d-33de-4b9d-a261-73ea2232af67 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f276a505d7 namespace: kserve-ci-e2e-test resourceVersion: "19526" uid: 395ba371-ef85-42af-9677-4f9ac88059a9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f279656fd8 namespace: kserve-ci-e2e-test resourceVersion: "19527" uid: 0d06f601-05f5-4a6e-acbc-47ce6416cab5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f279c616d2 namespace: kserve-ci-e2e-test resourceVersion: "19528" uid: 386579db-1a77-4d03-a66b-bd232e31740f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f279d0e01f namespace: kserve-ci-e2e-test resourceVersion: "19529" uid: fa256dd9-9ab1-404c-b36d-7c2d08a6d852 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f27c58f5db namespace: kserve-ci-e2e-test resourceVersion: "19532" uid: ea095a76-1a8d-42d8-88c7-e2c709556ff5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:04Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:04Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:17:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:04Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f27cc0b4d7 namespace: kserve-ci-e2e-test resourceVersion: "19533" uid: 995366c8-d989-4322-ac8c-2b0b3ad0b950 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:17:42Z" message: 'Readiness probe failed: dial tcp 10.133.0.28:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:17:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:42Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d627f2f43329a4 namespace: kserve-ci-e2e-test resourceVersion: "19848" uid: 6e4ce060-800d-4619-b30b-4362ba0e1d81 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d628028ca79767 namespace: kserve-ci-e2e-test resourceVersion: "20330" uid: 8c98f66b-847c-4471-b64d-d5fbc8919bba reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:18:13Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl namespace: kserve-ci-e2e-test resourceVersion: "19463" uid: 1f272975-7e98-408c-b98a-7ec39f09044e kind: Event lastTimestamp: "2026-09-17T16:18:13Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:18:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:18:13Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl.18d628028ca81b0a namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: b51cafbf-ea77-442e-8f0f-a845a8ca9178 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:03Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7c0d2-predictor-bbb4bd85b namespace: kserve-ci-e2e-test resourceVersion: "19442" uid: 45882a8a-e2ca-4f02-9169-3ace36ac73fe kind: Event lastTimestamp: "2026-09-17T16:17:03Z" message: 'Created pod: success-200-isvc-7c0d2-predictor-bbb4bd85b-kv5kl' metadata: creationTimestamp: "2026-09-17T16:17:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:03Z" name: success-200-isvc-7c0d2-predictor-bbb4bd85b.18d627f22e58591b namespace: kserve-ci-e2e-test resourceVersion: "19462" uid: 7e3a239c-67d2-4fc5-8c91-88875bdc6b15 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19438" uid: 45263d1f-1c96-4990-9e6b-53c2f6aa0807 kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: Scaled up replica set success-200-isvc-7c0d2-predictor-bbb4bd85b from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:17:02Z" name: success-200-isvc-7c0d2-predictor.18d627f228f6824c namespace: kserve-ci-e2e-test resourceVersion: "19445" uid: fc68961b-6561-473b-bd56-866594d7b5bf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:17:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 2163f3c9-8c58-4aa6-9ed5-1cf41ba2aba3 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: success-200-isvc-7c0d2-predictor.18d627f5a7c81340 namespace: kserve-ci-e2e-test resourceVersion: "20187" uid: 1d91ac6b-325c-4015-ab66-4ce323952ab8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-09-17T16:17:17Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c0d2-predictor namespace: kserve-ci-e2e-test resourceVersion: "19441" uid: 2163f3c9-8c58-4aa6-9ed5-1cf41ba2aba3 kind: Event lastTimestamp: "2026-09-17T16:18:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:17:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:18:03Z" name: success-200-isvc-7c0d2-predictor.18d627f5a851fd19 namespace: kserve-ci-e2e-test resourceVersion: "20192" uid: cd644032-3f83-4b3c-bb0b-84380f7512d8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: d58ac1bc-1b35-42bf-b023-1a117fa6c0bd kind: Event lastTimestamp: "2026-09-17T16:17:52Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:52Z" name: success-200-isvc-7c0d2.18d627f2291e652b namespace: kserve-ci-e2e-test resourceVersion: "20095" uid: a8c869cd-ba04-4e17-b44a-b06705c4dab7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: d58ac1bc-1b35-42bf-b023-1a117fa6c0bd kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: 'Failed to update status for InferenceService "success-200-isvc-7c0d2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:02Z" name: success-200-isvc-7c0d2.18d627f229c49607 namespace: kserve-ci-e2e-test resourceVersion: "19448" uid: f6753c4c-077b-4449-bd89-274788d1b6ec reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "19419" uid: d58ac1bc-1b35-42bf-b023-1a117fa6c0bd kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-7c0d2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:02Z" name: success-200-isvc-7c0d2.18d627f229c4e044 namespace: kserve-ci-e2e-test resourceVersion: "19449" uid: 09444a18-2a5c-4e30-ad28-d47514a62933 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:52Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c0d2 namespace: kserve-ci-e2e-test resourceVersion: "20092" uid: d58ac1bc-1b35-42bf-b023-1a117fa6c0bd kind: Event lastTimestamp: "2026-09-17T16:17:52Z" message: InferenceService [success-200-isvc-7c0d2] is Ready metadata: creationTimestamp: "2026-09-17T16:17:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:17:52Z" name: success-200-isvc-7c0d2.18d627fdb51b76db namespace: kserve-ci-e2e-test resourceVersion: "20093" uid: 8b691aa4-44a6-40bc-a3d0-85d7ec6731ac reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:27:28.697197Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24707" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j to ip-10-0-140-92.ec2.internal metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62883da740f0d namespace: kserve-ci-e2e-test resourceVersion: "24716" uid: db0ee27d-f415-4e14-a5d0-c63153c85f43 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7ce9a-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62883e6def06e namespace: kserve-ci-e2e-test resourceVersion: "24741" uid: 9236255b-76e5-4a5a-a759-13f9b8511698 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24721" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: Add eth0 [10.133.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:27:29Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d6288420690a92 namespace: kserve-ci-e2e-test resourceVersion: "24768" uid: 1f477cf7-aad5-4ff3-a166-239e15da0b10 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1941" already present on machine metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:29Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d628842211692f namespace: kserve-ci-e2e-test resourceVersion: "24770" uid: b743c7e7-2f69-4f9c-8789-eff094c82dc5 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:29Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d6288424d12e72 namespace: kserve-ci-e2e-test resourceVersion: "24771" uid: f93b80b2-12e9-4985-ba0c-d9098c8a8fce reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: Started container kserve-container metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:29Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62884253c3e65 namespace: kserve-ci-e2e-test resourceVersion: "24772" uid: 8997200f-d570-4c0f-8c6a-84d5c1a82719 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:29Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-09-17T16:27:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:29Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d6288425465a64 namespace: kserve-ci-e2e-test resourceVersion: "24773" uid: abb11c19-2c31-4fc9-971d-3bcdb9fb5d4c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62884285f8944 namespace: kserve-ci-e2e-test resourceVersion: "24774" uid: 98347ec1-437e-428e-baa8-a1526472bd49 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:27:30Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:27:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:27:30Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d6288428cb1c5d namespace: kserve-ci-e2e-test resourceVersion: "24775" uid: 2e5651db-2634-4cf3-a798-f8ab086a92af reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:27:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:28:07Z" message: 'Readiness probe failed: dial tcp 10.133.0.32:8080: connect: connection refused' metadata: creationTimestamp: "2026-09-17T16:27:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:07Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d6288497ab39aa namespace: kserve-ci-e2e-test resourceVersion: "25089" uid: ba0147df-9370-433b-8418-4f06b8747f8b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:28:38Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-09-17T16:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:38Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62894351f5f11 namespace: kserve-ci-e2e-test resourceVersion: "25577" uid: a02e7cf8-c7c4-4408-9006-d67efef729d2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j namespace: kserve-ci-e2e-test resourceVersion: "24712" uid: 440c5089-6096-468c-9246-dafd4c5aae11 kind: Event lastTimestamp: "2026-09-17T16:28:38Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-09-17T16:28:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:28:38Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j.18d62894351fde8c namespace: kserve-ci-e2e-test resourceVersion: "25579" uid: d9cd5ab6-c248-4e5b-961f-141b4677327b reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-92.ec2.internal source: component: kubelet host: ip-10-0-140-92.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7ce9a-predictor-85fc555b5d namespace: kserve-ci-e2e-test resourceVersion: "24693" uid: 6cbcafed-a222-437e-91f9-2d6f3ebbc748 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: 'Created pod: success-200-isvc-7ce9a-predictor-85fc555b5d-zzg4j' metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-7ce9a-predictor-85fc555b5d.18d62883d9c88319 namespace: kserve-ci-e2e-test resourceVersion: "24713" uid: 42da9052-6220-4754-bd94-2c4b7f2f82ad reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24691" uid: b6a7ca8a-6c7f-4fc8-a61e-18ca5df07010 kind: Event lastTimestamp: "2026-09-17T16:27:28Z" message: Scaled up replica set success-200-isvc-7ce9a-predictor-85fc555b5d from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:28Z" name: success-200-isvc-7ce9a-predictor.18d62883d505db76 namespace: kserve-ci-e2e-test resourceVersion: "24694" uid: bd6a3f21-f810-41ad-9e32-8a4e356bf8a7 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: a9345d2c-9fae-4203-8fdd-091b4d6b1630 kind: Event lastTimestamp: "2026-09-17T16:27:43Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:43Z" name: success-200-isvc-7ce9a-predictor.18d6288754128ab6 namespace: kserve-ci-e2e-test resourceVersion: "24904" uid: 24ec3d5b-d9d5-4e80-bbfd-8d4cc915bf16 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:27:43Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24692" uid: a9345d2c-9fae-4203-8fdd-091b4d6b1630 kind: Event lastTimestamp: "2026-09-17T16:27:43Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:27:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:27:43Z" name: success-200-isvc-7ce9a-predictor.18d6288754bbd841 namespace: kserve-ci-e2e-test resourceVersion: "24906" uid: 9d8f70b6-d86e-4b2d-b876-c56e05836122 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: a9345d2c-9fae-4203-8fdd-091b4d6b1630 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: success-200-isvc-7ce9a-predictor.18d6288ad393838e namespace: kserve-ci-e2e-test resourceVersion: "25421" uid: 4ca1d648-c0b7-4f8f-bb84-41710c00ecb1 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-09-17T16:27:58Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7ce9a-predictor namespace: kserve-ci-e2e-test resourceVersion: "24905" uid: a9345d2c-9fae-4203-8fdd-091b4d6b1630 kind: Event lastTimestamp: "2026-09-17T16:28:28Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:27:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:28:28Z" name: success-200-isvc-7ce9a-predictor.18d6288ad40d5d0a namespace: kserve-ci-e2e-test resourceVersion: "25425" uid: bc192387-68d8-4ee8-9786-8274be2aa7f6 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-09-17T16:27:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "24681" uid: b8ad1186-b1b8-42d6-af32-03b72d0028d1 kind: Event lastTimestamp: "2026-09-17T16:28:18Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-09-17T16:27:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:18Z" name: success-200-isvc-7ce9a.18d62883d50507be namespace: kserve-ci-e2e-test resourceVersion: "25334" uid: 67997480-433a-49a4-a94f-01c091ab0870 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:28:18Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7ce9a namespace: kserve-ci-e2e-test resourceVersion: "25332" uid: b8ad1186-b1b8-42d6-af32-03b72d0028d1 kind: Event lastTimestamp: "2026-09-17T16:28:18Z" message: InferenceService [success-200-isvc-7ce9a] is Ready metadata: creationTimestamp: "2026-09-17T16:28:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:28:18Z" name: success-200-isvc-7ce9a.18d6288f56859ced namespace: kserve-ci-e2e-test resourceVersion: "25333" uid: 24fc8582-6b77-4824-8dc9-6e878c1b5ca0 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:08:48.205214Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15907" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-0dabd-b566dcb87-59wr8 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d6277ef7ed2907 namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 6e1f81a3-e949-44c9-920f-515b6505e412 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15917" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: Add eth0 [10.132.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d6277f1a0b2903 namespace: kserve-ci-e2e-test resourceVersion: "15931" uid: 3de632e6-eef3-4f84-9871-d9b824a77db9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dabd} kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d6277f1bbcde34 namespace: kserve-ci-e2e-test resourceVersion: "15933" uid: a53707ab-35b1-475b-a044-e45183564fcd reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dabd} kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: 'Created container: switch-graph-0dabd' metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d6277f1e8a28d5 namespace: kserve-ci-e2e-test resourceVersion: "15934" uid: 28b247d5-dbba-4989-8c29-6acd74adf8ba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dabd} kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: Started container switch-graph-0dabd metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d6277f1efa528f namespace: kserve-ci-e2e-test resourceVersion: "15935" uid: f25a395a-a63f-4d10-943d-db9e65eeba44 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:17:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dabd} kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:17:02Z" message: Stopping container switch-graph-0dabd metadata: creationTimestamp: "2026-09-17T16:17:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:02Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d627f21f4a0195 namespace: kserve-ci-e2e-test resourceVersion: "19413" uid: f6baaa6c-38bc-4ed9-907d-34fc50062e3a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:17:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-0dabd} kind: Pod name: switch-graph-0dabd-b566dcb87-59wr8 namespace: kserve-ci-e2e-test resourceVersion: "15909" uid: 5eef1539-7576-4adc-ad2d-5c4135476663 kind: Event lastTimestamp: "2026-09-17T16:17:30Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:17:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:17:30Z" name: switch-graph-0dabd-b566dcb87-59wr8.18d627f2cc255eca namespace: kserve-ci-e2e-test resourceVersion: "19758" uid: f1c500f2-0b00-47ad-b55f-3834e8f953c2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-0dabd-b566dcb87 namespace: kserve-ci-e2e-test resourceVersion: "15903" uid: c02b3b96-b643-4f82-86ef-3253a008bf07 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: 'Created pod: switch-graph-0dabd-b566dcb87-59wr8' metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd-b566dcb87.18d6277ef725bc8e namespace: kserve-ci-e2e-test resourceVersion: "15908" uid: 1e60263e-81cd-44cc-a630-e7745fee23b0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15901" uid: bd37d82b-ed0d-4a91-8384-f06ab2644764 kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: Scaled up replica set switch-graph-0dabd-b566dcb87 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd.18d6277ef4fb1824 namespace: kserve-ci-e2e-test resourceVersion: "15905" uid: 2bde5e3b-e493-4fcf-90b4-b6a8e6f9424b reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:08:48Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15924" uid: 466849e2-ecff-4ca6-a159-1cde7df1c4aa kind: Event lastTimestamp: "2026-09-17T16:08:48Z" message: InferenceGraph [switch-graph-0dabd] is Ready metadata: creationTimestamp: "2026-09-17T16:08:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:08:48Z" name: switch-graph-0dabd.18d6277f01b290e5 namespace: kserve-ci-e2e-test resourceVersion: "15926" uid: b8bb26c5-5001-4187-974d-83523d8e9271 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 9bd404a1-5f1e-4379-abdb-14619a6c4067 kind: Event lastTimestamp: "2026-09-17T16:09:03Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:03Z" name: switch-graph-0dabd.18d6278274d37e65 namespace: kserve-ci-e2e-test resourceVersion: "16054" uid: ef9b8fa2-0e9c-40b4-a241-8618f905c513 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:09:03Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-0dabd namespace: kserve-ci-e2e-test resourceVersion: "15902" uid: 9bd404a1-5f1e-4379-abdb-14619a6c4067 kind: Event lastTimestamp: "2026-09-17T16:09:03Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-09-17T16:09:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:09:03Z" name: switch-graph-0dabd.18d6278275753278 namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 05e47eec-2dce-449a-957e-49ab1055cd1e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:29:08.860652Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-37c70-86c98d65b-tnx25 to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:29:08Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:29:08Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6289b2ca8eb1e namespace: kserve-ci-e2e-test resourceVersion: "25960" uid: 5fd03e41-188d-4d97-b756-bd8824118d2b reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25965" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: Add eth0 [10.132.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:29:09Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6289b4edfa87d namespace: kserve-ci-e2e-test resourceVersion: "25987" uid: 4ba355f9-9366-4612-b51a-322b44b32c6b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-37c70} kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" already present on machine metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:09Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6289b50e3b9b8 namespace: kserve-ci-e2e-test resourceVersion: "25989" uid: 347d83fd-d64e-4495-bde4-25913dc7cbc9 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-37c70} kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: 'Created container: switch-graph-37c70' metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:09Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6289b540e67d5 namespace: kserve-ci-e2e-test resourceVersion: "25990" uid: 6bb06d3d-ef6b-41e7-aa8c-15d25505247b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:09Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-37c70} kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:29:09Z" message: Started container switch-graph-37c70 metadata: creationTimestamp: "2026-09-17T16:29:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:29:09Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6289b5487f68a namespace: kserve-ci-e2e-test resourceVersion: "25991" uid: e6786ccd-ef82-4619-a911-7eefd166e5a9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:45:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-37c70} kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:45:28Z" message: Stopping container switch-graph-37c70 metadata: creationTimestamp: "2026-09-17T16:45:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:28Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6297f2e213565 namespace: kserve-ci-e2e-test resourceVersion: "33112" uid: fd361097-8205-4bff-9529-02b5b9555ef9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:45:31Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-37c70} kind: Pod name: switch-graph-37c70-86c98d65b-tnx25 namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: 28ef2274-6b65-469d-9491-19ccb33a1564 kind: Event lastTimestamp: "2026-09-17T16:45:56Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:45:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:45:56Z" name: switch-graph-37c70-86c98d65b-tnx25.18d6297fedbbd4a6 namespace: kserve-ci-e2e-test resourceVersion: "33395" uid: 92521c9b-eee2-46ea-8011-04a4e54c3998 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:08Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-37c70-86c98d65b namespace: kserve-ci-e2e-test resourceVersion: "25951" uid: 7a46e709-e5e3-456f-a58c-68c497236b5d kind: Event lastTimestamp: "2026-09-17T16:29:08Z" message: 'Created pod: switch-graph-37c70-86c98d65b-tnx25' metadata: creationTimestamp: "2026-09-17T16:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:08Z" name: switch-graph-37c70-86c98d65b.18d6289b2c174f43 namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: 2239e5ca-2214-4a32-9272-b926b8829597 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:08Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25948" uid: bd2efd2b-7c16-480d-af7f-5b23bf48f1d7 kind: Event lastTimestamp: "2026-09-17T16:29:08Z" message: Scaled up replica set switch-graph-37c70-86c98d65b from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:08Z" name: switch-graph-37c70.18d6289b2acda531 namespace: kserve-ci-e2e-test resourceVersion: "25954" uid: 8cd5b633-b7d2-44a3-b925-84f75a22f5d8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:29:08Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25966" uid: 7b27bd73-af29-4b89-b2fe-91c89033340f kind: Event lastTimestamp: "2026-09-17T16:29:08Z" message: InferenceGraph [switch-graph-37c70] is Ready metadata: creationTimestamp: "2026-09-17T16:29:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:29:08Z" name: switch-graph-37c70.18d6289b30195725 namespace: kserve-ci-e2e-test resourceVersion: "25968" uid: 34589020-3e0c-4007-aa7d-a36434e6f8c7 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:29:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: e0ac1364-500d-49da-a1df-5242b9856e1a kind: Event lastTimestamp: "2026-09-17T16:29:38Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:38Z" name: switch-graph-37c70.18d6289eaa75661f namespace: kserve-ci-e2e-test resourceVersion: "26228" uid: 44655ba3-5922-4270-802d-90888a8f1831 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-09-17T16:29:23Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-37c70 namespace: kserve-ci-e2e-test resourceVersion: "25949" uid: e0ac1364-500d-49da-a1df-5242b9856e1a kind: Event lastTimestamp: "2026-09-17T16:29:38Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-09-17T16:29:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:29:38Z" name: switch-graph-37c70.18d6289eab3d44bf namespace: kserve-ci-e2e-test resourceVersion: "26229" uid: a3a28323-d1aa-4032-9ece-20b1b7313980 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-09-17T16:07:37.829862Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14927" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-48b6f-889dd88b7-x6cqt to ip-10-0-128-222.ec2.internal metadata: creationTimestamp: "2026-09-17T16:07:37Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-09-17T16:07:37Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276e953a781c namespace: kserve-ci-e2e-test resourceVersion: "14933" uid: a3ad58cc-bf52-4adc-9993-3e3ad9e924de reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-76cc88db65-f6j8p source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:38Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14936" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:38Z" message: Add eth0 [10.132.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-09-17T16:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-09-17T16:07:38Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276eb8292efb namespace: kserve-ci-e2e-test resourceVersion: "14947" uid: 009a661c-96b8-459e-bbc4-d9c752ba76f2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:38Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" metadata: creationTimestamp: "2026-09-17T16:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:38Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276ebaed87a1 namespace: kserve-ci-e2e-test resourceVersion: "14949" uid: 59621ed7-71c6-4d4b-9c7b-184b1bb2704f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:41Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:977a2742e5f7b8ff67bbd8499ac42052de002d2d1c627b0a4860475854e68456" in 2.604s (2.604s including waiting). Image size: 211751511 bytes.' metadata: creationTimestamp: "2026-09-17T16:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:41Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276f562bf5d2 namespace: kserve-ci-e2e-test resourceVersion: "14967" uid: 03b25aab-51bd-418e-8d2e-fe25b5a1d730 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:41Z" message: 'Created container: switch-graph-48b6f' metadata: creationTimestamp: "2026-09-17T16:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:41Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276f5909d196 namespace: kserve-ci-e2e-test resourceVersion: "14968" uid: fbf26dfa-bd2c-4efd-b0b6-56aa4d16a63b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:41Z" message: Started container switch-graph-48b6f metadata: creationTimestamp: "2026-09-17T16:07:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:41Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d6276f59800ae4 namespace: kserve-ci-e2e-test resourceVersion: "14969" uid: 01621704-0dfd-4c41-a840-a1583fe1be78 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:47Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:07:47Z" message: Stopping container switch-graph-48b6f metadata: creationTimestamp: "2026-09-17T16:07:47Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:07:47Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d62770e840922a namespace: kserve-ci-e2e-test resourceVersion: "15056" uid: 3f940e8d-85b5-4b5a-acd4-79007e75d7d6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-09-17T16:07:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-48b6f} kind: Pod name: switch-graph-48b6f-889dd88b7-x6cqt namespace: kserve-ci-e2e-test resourceVersion: "14929" uid: dc43ce78-ee77-4449-b06c-d4ddb4da5e5b kind: Event lastTimestamp: "2026-09-17T16:08:17Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-09-17T16:07:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-09-17T16:08:17Z" name: switch-graph-48b6f-889dd88b7-x6cqt.18d62771fa7d4062 namespace: kserve-ci-e2e-test resourceVersion: "15427" uid: 48780a35-9937-4bd9-99b6-925b909550c4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-128-222.ec2.internal source: component: kubelet host: ip-10-0-128-222.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:37Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-48b6f-889dd88b7 namespace: kserve-ci-e2e-test resourceVersion: "14923" uid: e596a008-a85d-4152-b974-5b3e01a49f57 kind: Event lastTimestamp: "2026-09-17T16:07:37Z" message: 'Created pod: switch-graph-48b6f-889dd88b7-x6cqt' metadata: creationTimestamp: "2026-09-17T16:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:37Z" name: switch-graph-48b6f-889dd88b7.18d6276e9421759e namespace: kserve-ci-e2e-test resourceVersion: "14928" uid: d1c1bda4-ce31-4cf3-93f4-cad100c41f93 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:37Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-48b6f namespace: kserve-ci-e2e-test resourceVersion: "14921" uid: 36095dc1-d190-445c-a3eb-c843240489df kind: Event lastTimestamp: "2026-09-17T16:07:37Z" message: Scaled up replica set switch-graph-48b6f-889dd88b7 from 0 to 1 metadata: creationTimestamp: "2026-09-17T16:07:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-09-17T16:07:37Z" name: switch-graph-48b6f.18d6276e8f5f9b30 namespace: kserve-ci-e2e-test resourceVersion: "14924" uid: 7c80a63c-1fac-4033-b8de-dbcd25950faf reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-09-17T16:07:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-48b6f namespace: kserve-ci-e2e-test resourceVersion: "14940" uid: 74b3479f-90d3-4ab6-ba51-8cdea24afe4c kind: Event lastTimestamp: "2026-09-17T16:07:37Z" message: InferenceGraph [switch-graph-48b6f] is Ready metadata: creationTimestamp: "2026-09-17T16:07:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-09-17T16:07:38Z" name: switch-graph-48b6f.18d6276e9f56286f namespace: kserve-ci-e2e-test resourceVersion: "14942" uid: e7803f01-eed9-4092-8c96-7aec4c04e0f4 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal kind: EventList metadata: resourceVersion: "33405"