--- apiVersion: v1 items: - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:22:06.187915Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20514" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-daf0b-8f788765b-8f4h9 to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c6809fd0a9cb78 namespace: kserve-ci-e2e-test resourceVersion: "20520" uid: c02937c0-6cdf-4568-ae35-17d337b0eb24 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "ensemble-graph-daf0b-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c6809fda27b135 namespace: kserve-ci-e2e-test resourceVersion: "20562" uid: 3fe3b23b-b261-407f-b8a6-a65b3b4be167 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20526" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Add eth0 [10.133.0.23/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:22:07Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a00e6f8bb3 namespace: kserve-ci-e2e-test resourceVersion: "20590" uid: c4659044-7653-4c66-afcd-7d739223240d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-daf0b} kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a00fe0752e namespace: kserve-ci-e2e-test resourceVersion: "20593" uid: a2baf19b-5684-4a36-bc3a-0ce0bd716639 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-daf0b} kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: 'Created container: ensemble-graph-daf0b' metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a013581420 namespace: kserve-ci-e2e-test resourceVersion: "20597" uid: efeced88-0dc3-4c4c-a22a-a9356bf5cf18 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-daf0b} kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Started container ensemble-graph-daf0b metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a013d76aa0 namespace: kserve-ci-e2e-test resourceVersion: "20598" uid: a21f385a-bb9a-42e6-b57a-d44581f604d0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-daf0b} kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Stopping container ensemble-graph-daf0b metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a225b80963 namespace: kserve-ci-e2e-test resourceVersion: "20711" uid: c7bcdbb9-5c6f-4d7b-bdf6-5c74a809f753 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-daf0b} kind: Pod name: ensemble-graph-daf0b-8f788765b-8f4h9 namespace: kserve-ci-e2e-test resourceVersion: "20515" uid: 3b26ca8c-1441-43d1-a838-300c59815740 kind: Event lastTimestamp: "2026-07-28T16:22:44Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:44Z" name: ensemble-graph-daf0b-8f788765b-8f4h9.18c680a2dbe62201 namespace: kserve-ci-e2e-test resourceVersion: "21075" uid: 67f035bd-eb59-43c9-8217-70f402583007 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-daf0b-8f788765b namespace: kserve-ci-e2e-test resourceVersion: "20510" uid: d19f1e59-8bda-4959-924e-a3dbae9a7c8f kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Created pod: ensemble-graph-daf0b-8f788765b-8f4h9' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b-8f788765b.18c6809fd0403731 namespace: kserve-ci-e2e-test resourceVersion: "20516" uid: c0a4f3f2-29db-429b-a68a-f15b7fd92d26 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20502" uid: 7e1e9567-d3bb-4031-a051-eb7839ec5503 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Scaled up replica set ensemble-graph-daf0b-8f788765b from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b.18c6809fcf565e10 namespace: kserve-ci-e2e-test resourceVersion: "20513" uid: 68e14cbb-f873-4455-b3e6-71a9eaf29f6d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20540" uid: c27fc247-5e4f-4f0d-bd90-958a8da53695 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: InferenceGraph [ensemble-graph-daf0b] is Ready metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b.18c6809fd4685637 namespace: kserve-ci-e2e-test resourceVersion: "20542" uid: 9dbcf687-6b4f-4f4b-b8b4-6107d13cc89d reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: c27fc247-5e4f-4f0d-bd90-958a8da53695 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Failed to update status for InferenceGraph "ensemble-graph-daf0b": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b.18c6809fd81db18e namespace: kserve-ci-e2e-test resourceVersion: "20558" uid: 6aaba687-1b62-4475-953d-3f5eb4db1d7d reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20475" uid: c27fc247-5e4f-4f0d-bd90-958a8da53695 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "ensemble-graph-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:06Z" name: ensemble-graph-daf0b.18c6809fd81e1864 namespace: kserve-ci-e2e-test resourceVersion: "20560" uid: e6284c61-3474-4ea1-bb29-e20ddbf5b512 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:23:26.527272Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21570" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/ensemble-graph-e5aa9-657946d8cb-xvlqc to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c680b285432052 namespace: kserve-ci-e2e-test resourceVersion: "21580" uid: e7a4c451-5d9c-4906-a7be-0ec8ae5d6bf8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21586" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Add eth0 [10.133.0.25/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c680b29f542cd8 namespace: kserve-ci-e2e-test resourceVersion: "21629" uid: e59bc9b1-4645-4843-8f56-20a3ca565200 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e5aa9} kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c680b2a0c12478 namespace: kserve-ci-e2e-test resourceVersion: "21632" uid: a6c069a6-a87a-4120-8380-aa2834196453 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e5aa9} kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: 'Created container: ensemble-graph-e5aa9' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c680b2a3b1079f namespace: kserve-ci-e2e-test resourceVersion: "21636" uid: 4e7c4e0b-9ccd-4775-9532-1088fbbc5236 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e5aa9} kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Started container ensemble-graph-e5aa9 metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c680b2a42de86e namespace: kserve-ci-e2e-test resourceVersion: "21637" uid: 53070df0-79a6-45df-90ab-5b87f2e45b4c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e5aa9} kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Stopping container ensemble-graph-e5aa9 metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c68125acd9abaa namespace: kserve-ci-e2e-test resourceVersion: "25152" uid: 1e5ef184-718e-436e-9ddb-defd92cbecd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{ensemble-graph-e5aa9} kind: Pod name: ensemble-graph-e5aa9-657946d8cb-xvlqc namespace: kserve-ci-e2e-test resourceVersion: "21577" uid: ee779df5-c5a9-4762-9113-f8c2ebd538e4 kind: Event lastTimestamp: "2026-07-28T16:32:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:08Z" name: ensemble-graph-e5aa9-657946d8cb-xvlqc.18c681263f01e88a namespace: kserve-ci-e2e-test resourceVersion: "25492" uid: bc26da26-4d93-4058-9d2d-3479227e637b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: ensemble-graph-e5aa9-657946d8cb namespace: kserve-ci-e2e-test resourceVersion: "21564" uid: 9aba22a4-e130-4f0f-ba1d-e09a9543ea3c kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'Created pod: ensemble-graph-e5aa9-657946d8cb-xvlqc' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9-657946d8cb.18c680b283b8d486 namespace: kserve-ci-e2e-test resourceVersion: "21574" uid: c62c5bde-44bd-45ce-bba9-673b136c76d3 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21560" uid: dfa03d36-b339-49ea-a170-a8a086cfa9a4 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Scaled up replica set ensemble-graph-e5aa9-657946d8cb from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9.18c680b282581ae9 namespace: kserve-ci-e2e-test resourceVersion: "21565" uid: d184744d-e1d4-4e57-906f-6212bcc3191e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21598" uid: 58742b4c-414e-48b8-820e-3f64307c9396 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: InferenceGraph [ensemble-graph-e5aa9] is Ready metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:26Z" name: ensemble-graph-e5aa9.18c680b28a3c4f34 namespace: kserve-ci-e2e-test resourceVersion: "21603" uid: 41097de5-dae1-401f-b60a-b5215eba52b1 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: b75cec52-e8bc-413c-a3ef-1d8af6bdb2ff kind: Event lastTimestamp: "2026-07-28T16:23:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:41Z" name: ensemble-graph-e5aa9.18c680b60101ac84 namespace: kserve-ci-e2e-test resourceVersion: "21782" uid: 915b8572-8696-457c-a49e-0cfa87852590 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21562" uid: b75cec52-e8bc-413c-a3ef-1d8af6bdb2ff kind: Event lastTimestamp: "2026-07-28T16:23:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:41Z" name: ensemble-graph-e5aa9.18c680b601fda8c4 namespace: kserve-ci-e2e-test resourceVersion: "21784" uid: ab5eef41-8399-44ba-a993-c5de29e5d227 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: b75cec52-e8bc-413c-a3ef-1d8af6bdb2ff kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: ensemble-graph-e5aa9.18c680b980fbc026 namespace: kserve-ci-e2e-test resourceVersion: "21910" uid: 1c49443d-8eb5-4236-a069-191884ab7e0f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: ensemble-graph-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21783" uid: b75cec52-e8bc-413c-a3ef-1d8af6bdb2ff kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:23:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: ensemble-graph-e5aa9.18c680b981a24e69 namespace: kserve-ci-e2e-test resourceVersion: "21911" uid: 1127e578-7de7-4a69-baf0-8d005485afce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:11:41.318964Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15431" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e53924332 namespace: kserve-ci-e2e-test resourceVersion: "15435" uid: 991e26ef-fe32-4df2-9950-ac2d914427fb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-021cc-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e6068f0c0 namespace: kserve-ci-e2e-test resourceVersion: "15463" uid: 86f27025-2703-4078-bce2-9a7ac7a4c373 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15441" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: Add eth0 [10.134.0.34/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e91ff425b namespace: kserve-ci-e2e-test resourceVersion: "15483" uid: 0e60da0a-4ba6-443f-b07b-a31cd9b967a2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e93526dd3 namespace: kserve-ci-e2e-test resourceVersion: "15485" uid: 2896dabb-46d0-4a10-9992-e7cecaf7de4e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e96e02ca1 namespace: kserve-ci-e2e-test resourceVersion: "15491" uid: 3ed46bb9-8ca4-41e0-a4c8-be76ab4daa77 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e974f8430 namespace: kserve-ci-e2e-test resourceVersion: "15492" uid: 0fcf238f-b88f-476b-b545-292b1d2b8ed9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e975a9599 namespace: kserve-ci-e2e-test resourceVersion: "15493" uid: e0238c5c-1923-42fb-9d17-474d1c90883d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e9a76a1e5 namespace: kserve-ci-e2e-test resourceVersion: "15494" uid: 30868a88-41e0-487e-934e-7adc7d47abe6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:11:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:42Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800e9aeb20b5 namespace: kserve-ci-e2e-test resourceVersion: "15495" uid: 2cfa4213-7f12-4d09-9693-29cf8cf2f062 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:20:59Z" message: 'Readiness probe failed: dial tcp 10.134.0.34:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:59Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6800ed0e7a066 namespace: kserve-ci-e2e-test resourceVersion: "19909" uid: ca7f95f8-4a5a-4a86-9b89-13db1242078e reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6808f89a2e301 namespace: kserve-ci-e2e-test resourceVersion: "19844" uid: 4f2b595f-6143-4d42-a232-4f99a96c2ca6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c6808f89a60a01 namespace: kserve-ci-e2e-test resourceVersion: "19845" uid: b2b774f0-76a3-4ad3-9d1f-ab79ec2739b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4 namespace: kserve-ci-e2e-test resourceVersion: "15432" uid: 4f39f911-e715-46b1-b23e-50899a865a99 kind: Event lastTimestamp: "2026-07-28T16:20:59Z" message: 'Readiness probe failed: Get "https://10.134.0.34:8643/healthz": dial tcp 10.134.0.34:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:20:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:59Z" name: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4.18c680904557fedc namespace: kserve-ci-e2e-test resourceVersion: "19908" uid: c4235dfe-9e05-4891-96c9-75b473482f0c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-021cc-predictor-9cd49ff5b namespace: kserve-ci-e2e-test resourceVersion: "15423" uid: 149d51e7-e045-41b3-ac81-317aa68e7d97 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'Created pod: error-404-isvc-021cc-predictor-9cd49ff5b-nhxs4' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc-predictor-9cd49ff5b.18c6800e533e1b06 namespace: kserve-ci-e2e-test resourceVersion: "15433" uid: 0f36a184-d6bb-41db-a462-1c271adc5459 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15420" uid: f87ce506-2cda-40e5-8d69-f280479e7a13 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Scaled up replica set error-404-isvc-021cc-predictor-9cd49ff5b from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc-predictor.18c6800e51c43b59 namespace: kserve-ci-e2e-test resourceVersion: "15425" uid: f725059d-3aae-4b05-bb1c-635c9215842c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 19e484ee-d0f2-43f9-955a-8e77bd766e15 kind: Event lastTimestamp: "2026-07-28T16:11:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:56Z" name: error-404-isvc-021cc-predictor.18c68011d0705d79 namespace: kserve-ci-e2e-test resourceVersion: "15625" uid: e4760619-f5fd-4280-a643-415175836549 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15422" uid: 19e484ee-d0f2-43f9-955a-8e77bd766e15 kind: Event lastTimestamp: "2026-07-28T16:11:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:56Z" name: error-404-isvc-021cc-predictor.18c68011d0b02705 namespace: kserve-ci-e2e-test resourceVersion: "15628" uid: 78f06f51-f4be-42fa-9b0b-1e81cc21bfe0 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 19e484ee-d0f2-43f9-955a-8e77bd766e15 kind: Event lastTimestamp: "2026-07-28T16:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:11Z" name: error-404-isvc-021cc-predictor.18c680154f9bd862 namespace: kserve-ci-e2e-test resourceVersion: "16494" uid: 6d4e7261-9822-4a6b-9e50-a8281b6fd551 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:12:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15626" uid: 19e484ee-d0f2-43f9-955a-8e77bd766e15 kind: Event lastTimestamp: "2026-07-28T16:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:12:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:11Z" name: error-404-isvc-021cc-predictor.18c68015501b9eb0 namespace: kserve-ci-e2e-test resourceVersion: "16495" uid: b6761778-6af5-46f7-8d6d-6fb653150f4e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: fc13a12b-4ed6-40a5-99a3-14c39955de9c kind: Event lastTimestamp: "2026-07-28T16:12:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:29Z" name: error-404-isvc-021cc.18c6800e51fa8298 namespace: kserve-ci-e2e-test resourceVersion: "15930" uid: 9f57d3e7-c9a4-499d-9549-4f4917493e6a reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: fc13a12b-4ed6-40a5-99a3-14c39955de9c kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-021cc": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-021cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc.18c6800e52657f47 namespace: kserve-ci-e2e-test resourceVersion: "15428" uid: e8746128-82aa-449f-86ec-a4462f84eb41 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15403" uid: fc13a12b-4ed6-40a5-99a3-14c39955de9c kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-021cc": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-021cc.18c6800e5265d643 namespace: kserve-ci-e2e-test resourceVersion: "15430" uid: dcd64d6e-3238-4c7b-9149-f6430ed989cf reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15928" uid: fc13a12b-4ed6-40a5-99a3-14c39955de9c kind: Event lastTimestamp: "2026-07-28T16:12:29Z" message: InferenceService [error-404-isvc-021cc] is Ready metadata: creationTimestamp: "2026-07-28T16:12:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:29Z" name: error-404-isvc-021cc.18c680198dd5db55 namespace: kserve-ci-e2e-test resourceVersion: "15929" uid: c4e16db2-2050-4920-a885-94d3e80b5790 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:12:51.412627Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16274" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-0402e-predictor-74b6c76776-g66wb to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ea57a9c48 namespace: kserve-ci-e2e-test resourceVersion: "16278" uid: 3c60bca0-eab1-4432-87b2-c2ea001119d6 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16285" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Add eth0 [10.134.0.38/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ec0124e0a namespace: kserve-ci-e2e-test resourceVersion: "16304" uid: 03fd7031-d856-44a7-b1f8-ca62744fddcc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ec20a3865 namespace: kserve-ci-e2e-test resourceVersion: "16308" uid: 39dc5b8a-e283-464f-8029-01292a558a94 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ec57ca6d0 namespace: kserve-ci-e2e-test resourceVersion: "16310" uid: 4a582c08-4b01-4b89-9047-012bc3577266 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ec6094ce3 namespace: kserve-ci-e2e-test resourceVersion: "16311" uid: 9d9f6e84-9a75-4cc2-82d5-266118f7091e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ec613b2a1 namespace: kserve-ci-e2e-test resourceVersion: "16312" uid: 085f76a8-2758-417e-b97d-1b7b2cb387c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:52Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801eca67fbec namespace: kserve-ci-e2e-test resourceVersion: "16314" uid: e1255838-52cc-4e14-b051-25b9c2d57e80 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:12:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:52Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ecae9ccef namespace: kserve-ci-e2e-test resourceVersion: "16315" uid: f7f9da6a-63d3-4b46-b29a-0e440efd373a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:22:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.38:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:08Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6801ef6ce433f namespace: kserve-ci-e2e-test resourceVersion: "20623" uid: 742d0c37-62da-48db-a980-23e1bf67c08f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6809fce1e5fa7 namespace: kserve-ci-e2e-test resourceVersion: "20506" uid: 89f5f5bc-7ecc-47dd-971d-4eb5f229275f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c6809fce1fb23b namespace: kserve-ci-e2e-test resourceVersion: "20508" uid: 69f912b7-2b65-4467-939c-ac5d79376ac4 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-0402e-predictor-74b6c76776-g66wb namespace: kserve-ci-e2e-test resourceVersion: "16277" uid: 675be705-9def-42cb-983c-5a12e49fe04a kind: Event lastTimestamp: "2026-07-28T16:22:08Z" message: 'Readiness probe failed: Get "https://10.134.0.38:8643/healthz": dial tcp 10.134.0.38:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:08Z" name: error-404-isvc-0402e-predictor-74b6c76776-g66wb.18c680a06b613d2a namespace: kserve-ci-e2e-test resourceVersion: "20622" uid: 307ce577-53af-4376-9682-cea4819f3085 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-0402e-predictor-74b6c76776 namespace: kserve-ci-e2e-test resourceVersion: "16268" uid: 60547a32-cb2a-47f7-99d3-606a6293ca43 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: 'Created pod: error-404-isvc-0402e-predictor-74b6c76776-g66wb' metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor-74b6c76776.18c6801ea4b81edf namespace: kserve-ci-e2e-test resourceVersion: "16275" uid: 3edc4fed-26ff-435c-a371-e3a9c902a4d9 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16265" uid: 9d0f115f-8b90-439d-9d95-6d332d4b086d kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Scaled up replica set error-404-isvc-0402e-predictor-74b6c76776 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:51Z" name: error-404-isvc-0402e-predictor.18c6801ea2436daf namespace: kserve-ci-e2e-test resourceVersion: "16270" uid: 099fa1f9-eb94-4f56-af95-ce7463fd3088 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: e6dcdb85-6fcc-4abe-ac9a-fb64e366ef39 kind: Event lastTimestamp: "2026-07-28T16:13:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:21Z" name: error-404-isvc-0402e-predictor.18c6802220fb4adc namespace: kserve-ci-e2e-test resourceVersion: "16575" uid: bc5428f3-ec24-4119-9842-97549ad46f72 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16267" uid: e6dcdb85-6fcc-4abe-ac9a-fb64e366ef39 kind: Event lastTimestamp: "2026-07-28T16:13:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:21Z" name: error-404-isvc-0402e-predictor.18c680222176d4d1 namespace: kserve-ci-e2e-test resourceVersion: "16576" uid: 4d7ed129-edd7-4f1b-91bd-278b3a8ae233 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: e6dcdb85-6fcc-4abe-ac9a-fb64e366ef39 kind: Event lastTimestamp: "2026-07-28T16:14:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:14:06Z" name: error-404-isvc-0402e-predictor.18c680291f109bee namespace: kserve-ci-e2e-test resourceVersion: "16954" uid: b1d2f196-8992-49f9-ab2a-96e47e61a049 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:13:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16451" uid: e6dcdb85-6fcc-4abe-ac9a-fb64e366ef39 kind: Event lastTimestamp: "2026-07-28T16:14:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:14:06Z" name: error-404-isvc-0402e-predictor.18c680291f775f50 namespace: kserve-ci-e2e-test resourceVersion: "16955" uid: 7ecf429e-a3b6-4ea3-a1a4-55d24381ba8c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0402e namespace: kserve-ci-e2e-test resourceVersion: "16233" uid: ee1f93b4-2818-469b-9dee-342c3e8021df kind: Event lastTimestamp: "2026-07-28T16:13:38Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:13:38Z" name: error-404-isvc-0402e.18c6801ea2bddffd namespace: kserve-ci-e2e-test resourceVersion: "16718" uid: d773348d-dbdd-4f6c-9c88-986250363b53 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-0402e namespace: kserve-ci-e2e-test resourceVersion: "16712" uid: ee1f93b4-2818-469b-9dee-342c3e8021df kind: Event lastTimestamp: "2026-07-28T16:13:38Z" message: InferenceService [error-404-isvc-0402e] is Ready metadata: creationTimestamp: "2026-07-28T16:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:13:38Z" name: error-404-isvc-0402e.18c68029b409c10d namespace: kserve-ci-e2e-test resourceVersion: "16713" uid: 17cdec64-0dfa-472b-828b-2f4a27f867ff reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:10:07.385451Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14388" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-3692d-predictor-97c56c7cd-jjssk to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:10:07Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ff874b2a12b namespace: kserve-ci-e2e-test resourceVersion: "14392" uid: e5c0d49b-2d5f-4a85-8863-db3eb58e87ed reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14397" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Add eth0 [10.134.0.31/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:10:07Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ff88f0e150a namespace: kserve-ci-e2e-test resourceVersion: "14443" uid: 2e3eaf37-cf73-4ced-95b6-a6a9796b6b9c reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Pulling image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ff890780a9e namespace: kserve-ci-e2e-test resourceVersion: "14446" uid: 83ace21f-87c9-4dfc-bf38-bf220bbc6236 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: 'Successfully pulled image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" in 15.874s (15.874s including waiting). Image size: 1333941339 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffc42a89437 namespace: kserve-ci-e2e-test resourceVersion: "14628" uid: 54f2ab53-b96f-4e6e-bb2d-96a03c9b2c47 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffc46fa6690 namespace: kserve-ci-e2e-test resourceVersion: "14630" uid: 58d323b5-9c67-48c4-aa11-e02f36187be6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffc477d38ef namespace: kserve-ci-e2e-test resourceVersion: "14632" uid: 8edbc611-5d43-4d43-a612-508a3dd7f121 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffc478a724e namespace: kserve-ci-e2e-test resourceVersion: "14634" uid: 91d045c7-cbfa-440c-9584-5f015548ec9e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.69s (2.69s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffce7ec2567 namespace: kserve-ci-e2e-test resourceVersion: "14656" uid: b03c6464-80c3-40e8-9f5b-4a9ebc37ad2e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffcecacaad4 namespace: kserve-ci-e2e-test resourceVersion: "14658" uid: 2f80b3c3-e7b4-44bd-aa40-349fcc7f0ee8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffced3275b6 namespace: kserve-ci-e2e-test resourceVersion: "14660" uid: 57453994-c144-492b-8fea-367f4d6c6972 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:10:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:11:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.31:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:10:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:43Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c67ffd0825b2da namespace: kserve-ci-e2e-test resourceVersion: "15501" uid: 48817495-9c36-4003-865e-29cb473d5253 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c6800e5cdd226a namespace: kserve-ci-e2e-test resourceVersion: "15459" uid: 709fc113-6c90-422a-86e5-68d0815b94b2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c6800e5cdd9d8a namespace: kserve-ci-e2e-test resourceVersion: "15460" uid: e36eb1ad-18aa-4fcf-b23e-e821195d792a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk namespace: kserve-ci-e2e-test resourceVersion: "14390" uid: 48d09874-be09-45a1-8c27-a26033be48a6 kind: Event lastTimestamp: "2026-07-28T16:11:43Z" message: 'Readiness probe failed: Get "https://10.134.0.31:8643/healthz": dial tcp 10.134.0.31:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:11:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:43Z" name: error-404-isvc-3692d-predictor-97c56c7cd-jjssk.18c6800eba85eb76 namespace: kserve-ci-e2e-test resourceVersion: "15500" uid: 3b56f478-5427-45e1-87a3-3aa0508fdaf5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-3692d-predictor-97c56c7cd namespace: kserve-ci-e2e-test resourceVersion: "14381" uid: 024497a5-d63a-47ff-8c32-c93989a73714 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'Created pod: error-404-isvc-3692d-predictor-97c56c7cd-jjssk' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: error-404-isvc-3692d-predictor-97c56c7cd.18c67ff8744117f6 namespace: kserve-ci-e2e-test resourceVersion: "14389" uid: 6659ed9f-e609-45f6-a57b-4eb1a7cdb50a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14379" uid: d3cdfd68-d8a3-4e97-a6cc-172295ac35ea kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Scaled up replica set error-404-isvc-3692d-predictor-97c56c7cd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: error-404-isvc-3692d-predictor.18c67ff8727797b2 namespace: kserve-ci-e2e-test resourceVersion: "14383" uid: ff5d1242-bc92-4d0b-b213-0f2d541c5b83 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: b416c104-6a5c-4b3f-b785-7b893c858853 kind: Event lastTimestamp: "2026-07-28T16:11:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:07Z" name: error-404-isvc-3692d-predictor.18c67ffbf10ce196 namespace: kserve-ci-e2e-test resourceVersion: "15024" uid: 9a454ef3-8af7-46d8-92f3-1ea181176015 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14380" uid: b416c104-6a5c-4b3f-b785-7b893c858853 kind: Event lastTimestamp: "2026-07-28T16:11:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:07Z" name: error-404-isvc-3692d-predictor.18c67ffbf1563deb namespace: kserve-ci-e2e-test resourceVersion: "15025" uid: bafc245b-8552-4412-88b1-0192e825fb81 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: b416c104-6a5c-4b3f-b785-7b893c858853 kind: Event lastTimestamp: "2026-07-28T16:11:37Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:37Z" name: error-404-isvc-3692d-predictor.18c68009ed4fd592 namespace: kserve-ci-e2e-test resourceVersion: "15321" uid: 839212c2-0df0-4450-aa5d-9f7a4f80a7d8 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14613" uid: b416c104-6a5c-4b3f-b785-7b893c858853 kind: Event lastTimestamp: "2026-07-28T16:11:37Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:37Z" name: error-404-isvc-3692d-predictor.18c68009ed9826a3 namespace: kserve-ci-e2e-test resourceVersion: "15322" uid: 2ff47662-ad40-4253-9e38-b33c035bf738 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "14362" uid: a43e89f2-c8fd-4075-a7d6-735084cc786f kind: Event lastTimestamp: "2026-07-28T16:11:13Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:13Z" name: error-404-isvc-3692d.18c67ff8727ffad3 namespace: kserve-ci-e2e-test resourceVersion: "15078" uid: 7d7e91ff-d703-4646-9c11-2eb7b937f056 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:13Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "15076" uid: a43e89f2-c8fd-4075-a7d6-735084cc786f kind: Event lastTimestamp: "2026-07-28T16:11:13Z" message: InferenceService [error-404-isvc-3692d] is Ready metadata: creationTimestamp: "2026-07-28T16:11:13Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:13Z" name: error-404-isvc-3692d.18c68007c3d0cac8 namespace: kserve-ci-e2e-test resourceVersion: "15077" uid: c60e4d4d-076e-4078-ac9a-37b22890aebc reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:51.732055Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26413" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-49c25-predictor-5c44655c88-2xw8x to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681361e17a5e5 namespace: kserve-ci-e2e-test resourceVersion: "26417" uid: d0546f4b-1ca8-4d0d-9895-3ea52943ceeb reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26423" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Add eth0 [10.134.0.52/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c6813639406ee3 namespace: kserve-ci-e2e-test resourceVersion: "26438" uid: 9966523d-cb15-4dbc-8c4f-e2d6af2501e3 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681363aad07e4 namespace: kserve-ci-e2e-test resourceVersion: "26440" uid: 0a74b646-703e-48e1-9601-c6a0bbd6a737 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681363da5ea3d namespace: kserve-ci-e2e-test resourceVersion: "26444" uid: d2e7441e-53a6-47db-8c36-294a0dc56fba reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681363e211e77 namespace: kserve-ci-e2e-test resourceVersion: "26445" uid: 4b6860da-4035-4d56-ac15-78b45293562f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681363e2c638c namespace: kserve-ci-e2e-test resourceVersion: "26446" uid: 6f87e7ac-ee86-41f6-b7d1-aeeb454ce3f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c68136417639ec namespace: kserve-ci-e2e-test resourceVersion: "26447" uid: 026ccb8e-279e-4267-a9e1-66e2ad6d28b4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c6813641f77a03 namespace: kserve-ci-e2e-test resourceVersion: "26448" uid: b469927c-dd60-44a4-8b77-8c13553c52c0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:42:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.52:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:19Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c6813699e019cb namespace: kserve-ci-e2e-test resourceVersion: "30623" uid: 08f4ff38-6f1d-4848-a504-f86bc9d853c3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:42:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:16Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681b9a21f424d namespace: kserve-ci-e2e-test resourceVersion: "30595" uid: b7dd7e22-8ea3-46f8-b197-24f16ea964c0 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:42:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:16Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681b9a2206607 namespace: kserve-ci-e2e-test resourceVersion: "30596" uid: 185eacfd-33ad-4b72-aa31-00769455aa0a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x namespace: kserve-ci-e2e-test resourceVersion: "26415" uid: 93032262-4d31-45cb-9ad2-161cab3accbd kind: Event lastTimestamp: "2026-07-28T16:42:19Z" message: 'Readiness probe failed: Get "https://10.134.0.52:8643/healthz": dial tcp 10.134.0.52:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:42:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:19Z" name: error-404-isvc-49c25-predictor-5c44655c88-2xw8x.18c681ba62537303 namespace: kserve-ci-e2e-test resourceVersion: "30622" uid: 0fbe420e-49de-4cfb-8c5a-b42b52dbae48 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-49c25-predictor-5c44655c88 namespace: kserve-ci-e2e-test resourceVersion: "26401" uid: a300a73b-379b-4fd9-b6fa-6089ecc22d8a kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'Created pod: error-404-isvc-49c25-predictor-5c44655c88-2xw8x' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-49c25-predictor-5c44655c88.18c681361da8843d namespace: kserve-ci-e2e-test resourceVersion: "26414" uid: 231028f8-c164-46f5-ac0e-f957de1e2528 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26394" uid: 67070fb1-5293-4ac2-99a5-d9e4d59c55ce kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Scaled up replica set error-404-isvc-49c25-predictor-5c44655c88 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-49c25-predictor.18c681361bc3d777 namespace: kserve-ci-e2e-test resourceVersion: "26404" uid: c398e186-7ae0-4290-9c20-27fb6ae1edf0 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: 3f7eaa5a-0f7a-4294-b267-2641b2320cb6 kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: error-404-isvc-49c25-predictor.18c681399a50b415 namespace: kserve-ci-e2e-test resourceVersion: "26721" uid: 9adcf5d5-6e67-4203-86e7-36f931a80f41 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26400" uid: 3f7eaa5a-0f7a-4294-b267-2641b2320cb6 kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: error-404-isvc-49c25-predictor.18c681399ab872d7 namespace: kserve-ci-e2e-test resourceVersion: "26722" uid: 74340c0e-90a8-48d8-9a63-a53ccc4bc027 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 3f7eaa5a-0f7a-4294-b267-2641b2320cb6 kind: Event lastTimestamp: "2026-07-28T16:34:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:06Z" name: error-404-isvc-49c25-predictor.18c68140987f6124 namespace: kserve-ci-e2e-test resourceVersion: "27177" uid: 67e410eb-1073-45ef-ae40-7c1ca7f3f6bc reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26591" uid: 3f7eaa5a-0f7a-4294-b267-2641b2320cb6 kind: Event lastTimestamp: "2026-07-28T16:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:06Z" name: error-404-isvc-49c25-predictor.18c6814098ec3810 namespace: kserve-ci-e2e-test resourceVersion: "27178" uid: 35fb2872-2104-44b8-81d2-c92bbebf63bd reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: bb864d56-6454-4d5c-a389-69a35815c710 kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-49c25.18c681361be02f08 namespace: kserve-ci-e2e-test resourceVersion: "26887" uid: a48f8958-53aa-4aaf-9d21-45be7d53f895 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: bb864d56-6454-4d5c-a389-69a35815c710 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'Failed to update status for InferenceService "error-404-isvc-49c25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-49c25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-49c25.18c681361c87c977 namespace: kserve-ci-e2e-test resourceVersion: "26408" uid: 71346590-4c85-4227-b36f-c96728b821c5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26373" uid: bb864d56-6454-4d5c-a389-69a35815c710 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-49c25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-49c25.18c681361c881ffb namespace: kserve-ci-e2e-test resourceVersion: "26411" uid: f3977993-1dff-436b-803e-697e236171f4 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26884" uid: bb864d56-6454-4d5c-a389-69a35815c710 kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: InferenceService [error-404-isvc-49c25] is Ready metadata: creationTimestamp: "2026-07-28T16:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-49c25.18c681415723def5 namespace: kserve-ci-e2e-test resourceVersion: "26885" uid: 3d90f011-07ce-4171-9eac-e5e1fcfba0c1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:22:06.284548Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20547" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809fd66c4d2c namespace: kserve-ci-e2e-test resourceVersion: "20553" uid: 0e6d1757-dc71-4558-8fe7-9a58ff520370 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20557" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Add eth0 [10.134.0.42/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ff14a9e48 namespace: kserve-ci-e2e-test resourceVersion: "20576" uid: b5ef08b2-106e-4cfe-b406-06a07eab776e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ff2d1066e namespace: kserve-ci-e2e-test resourceVersion: "20578" uid: bcb840d7-ace5-4279-89e0-534840768b8b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ff6b331e4 namespace: kserve-ci-e2e-test resourceVersion: "20581" uid: a1b98bef-e3b7-49af-bddb-ddad0c4a11d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ff7549d63 namespace: kserve-ci-e2e-test resourceVersion: "20582" uid: 93be3eb2-37b7-41f7-9658-a10ce21d7a56 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ff75ef91a namespace: kserve-ci-e2e-test resourceVersion: "20583" uid: 27e9d425-9bcf-49b1-aac6-0dd1b35e3265 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ffa65d400 namespace: kserve-ci-e2e-test resourceVersion: "20585" uid: 9fa783ec-86fe-4374-ad01-30addd51fdbc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c6809ffae965dd namespace: kserve-ci-e2e-test resourceVersion: "20586" uid: f164ddfb-cd01-4003-a7fc-b77a73a8bd94 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:22:53Z" message: 'Readiness probe failed: dial tcp 10.134.0.42:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:53Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c680a01e3b137f namespace: kserve-ci-e2e-test resourceVersion: "21159" uid: 86ab89df-7ccb-473b-b81d-2e0d08dbfdf3 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c680b2784ebaa0 namespace: kserve-ci-e2e-test resourceVersion: "21512" uid: be9bb8f5-fd63-450e-abe8-2f594455a4cc reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c680b2784f7435 namespace: kserve-ci-e2e-test resourceVersion: "21513" uid: 4e7cb0da-8780-48b3-baae-0879d674d865 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb namespace: kserve-ci-e2e-test resourceVersion: "20551" uid: 3534c90a-ca82-4e2c-8964-cd2439ba2fee kind: Event lastTimestamp: "2026-07-28T16:23:28Z" message: 'Readiness probe failed: Get "https://10.134.0.42:8643/healthz": dial tcp 10.134.0.42:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:28Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb.18c680b2fa929832 namespace: kserve-ci-e2e-test resourceVersion: "21667" uid: f9ac402f-47b9-47c7-9c8c-cde7456f7fcf reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-7c1b1-predictor-7f58bd6c46 namespace: kserve-ci-e2e-test resourceVersion: "20535" uid: 29f67244-b801-40b5-881e-58fc0d5e4cb4 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Created pod: error-404-isvc-7c1b1-predictor-7f58bd6c46-mskvb' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor-7f58bd6c46.18c6809fd5d49279 namespace: kserve-ci-e2e-test resourceVersion: "20549" uid: d1a729a9-b9b5-4ad9-9c78-a5438f2ecf48 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20533" uid: 62eea5f2-bedb-4300-b5f7-55d1e703effd kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Scaled up replica set error-404-isvc-7c1b1-predictor-7f58bd6c46 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1-predictor.18c6809fd39aa736 namespace: kserve-ci-e2e-test resourceVersion: "20538" uid: 7158814b-9c23-40fc-8b37-59e9b7444222 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:22:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 4aee1e53-da86-4293-bbf2-2dc9d2f2f7c1 kind: Event lastTimestamp: "2026-07-28T16:22:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:36Z" name: error-404-isvc-7c1b1-predictor.18c680a352106594 namespace: kserve-ci-e2e-test resourceVersion: "21016" uid: 25f2661a-7c45-4602-a287-6367f72a9581 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:22:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20534" uid: 4aee1e53-da86-4293-bbf2-2dc9d2f2f7c1 kind: Event lastTimestamp: "2026-07-28T16:22:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:36Z" name: error-404-isvc-7c1b1-predictor.18c680a3527f8026 namespace: kserve-ci-e2e-test resourceVersion: "21017" uid: d81e8ebb-ce2f-4d23-b4bd-f2dfa58d555a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 4aee1e53-da86-4293-bbf2-2dc9d2f2f7c1 kind: Event lastTimestamp: "2026-07-28T16:23:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:21Z" name: error-404-isvc-7c1b1-predictor.18c680aa50113a16 namespace: kserve-ci-e2e-test resourceVersion: "21439" uid: e3f00bff-558a-472e-aa50-db0d8c9cb065 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20882" uid: 4aee1e53-da86-4293-bbf2-2dc9d2f2f7c1 kind: Event lastTimestamp: "2026-07-28T16:23:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:21Z" name: error-404-isvc-7c1b1-predictor.18c680aa509ace65 namespace: kserve-ci-e2e-test resourceVersion: "21440" uid: 67d80b70-6a0f-4b89-848f-705cf866e991 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 69bd49d9-e865-40ff-a233-c933330ac948 kind: Event lastTimestamp: "2026-07-28T16:23:03Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:03Z" name: error-404-isvc-7c1b1.18c6809fd3fd2814 namespace: kserve-ci-e2e-test resourceVersion: "21244" uid: 7de78085-0343-4ba3-b865-5c25f87f5322 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 69bd49d9-e865-40ff-a233-c933330ac948 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Failed to update status for InferenceService "error-404-isvc-7c1b1": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c1b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1.18c6809fd4748575 namespace: kserve-ci-e2e-test resourceVersion: "20544" uid: 26be3a16-e02b-405e-8d36-9ae5c7582ede reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "20504" uid: 69bd49d9-e865-40ff-a233-c933330ac948 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-7c1b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:06Z" name: error-404-isvc-7c1b1.18c6809fd474d26a namespace: kserve-ci-e2e-test resourceVersion: "20546" uid: 2a2db5f9-854d-46df-ab45-99e0c3f2a7ad reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:03Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21241" uid: 69bd49d9-e865-40ff-a233-c933330ac948 kind: Event lastTimestamp: "2026-07-28T16:23:03Z" message: InferenceService [error-404-isvc-7c1b1] is Ready metadata: creationTimestamp: "2026-07-28T16:23:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:03Z" name: error-404-isvc-7c1b1.18c680ad2e3aced3 namespace: kserve-ci-e2e-test resourceVersion: "21242" uid: 83455b3b-a378-4c5d-91e6-f01c364f2c8a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:31:41.478515Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25220" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-a9ea4-predictor-756b95979d-vf9xt to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125c2a7a468 namespace: kserve-ci-e2e-test resourceVersion: "25224" uid: c879526b-6218-4f87-b671-4a8b1f0fa498 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25236" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Add eth0 [10.134.0.48/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125dd8923e7 namespace: kserve-ci-e2e-test resourceVersion: "25262" uid: 5244f6c5-8058-4965-ac42-dd594754c048 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125df783bdc namespace: kserve-ci-e2e-test resourceVersion: "25265" uid: bec2a60d-83c3-4d76-b506-1c989841c41d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:42Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125e3109216 namespace: kserve-ci-e2e-test resourceVersion: "25270" uid: 5d868d1a-3a4e-4d78-a251-3226d1e5bc5b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:42Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125e394e75d namespace: kserve-ci-e2e-test resourceVersion: "25272" uid: e4c690f1-9217-4f42-a111-bb371c807d9f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:42Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125e3a043b3 namespace: kserve-ci-e2e-test resourceVersion: "25273" uid: 8d87a2d9-2640-4818-bb8d-1384594ed131 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:42Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125e6bf6db4 namespace: kserve-ci-e2e-test resourceVersion: "25274" uid: b72760c8-a98c-4119-aaab-24b9deb32795 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:31:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:31:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:42Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68125e7428eae namespace: kserve-ci-e2e-test resourceVersion: "25275" uid: 6c094d01-66e0-42c2-bc73-cd925e1f604d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.48:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c681263680c0cd namespace: kserve-ci-e2e-test resourceVersion: "25855" uid: 537259bb-7ae2-402e-b323-29e06050203f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c681361c61317b namespace: kserve-ci-e2e-test resourceVersion: "26407" uid: c768102c-75e2-4d82-8eb1-4a3c2a06dc48 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:51Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c681361c623811 namespace: kserve-ci-e2e-test resourceVersion: "26409" uid: 2503ea7b-4d05-4500-988f-68221af58f81 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt namespace: kserve-ci-e2e-test resourceVersion: "25222" uid: 1f6b0fb6-302a-4184-b950-ec0c0ed7f4e0 kind: Event lastTimestamp: "2026-07-28T16:32:54Z" message: 'Readiness probe failed: Get "https://10.134.0.48:8643/healthz": dial tcp 10.134.0.48:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:54Z" name: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt.18c68136bea9e458 namespace: kserve-ci-e2e-test resourceVersion: "26478" uid: 1160ea5f-a23b-43ce-8f01-40fbd208aec1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-a9ea4-predictor-756b95979d namespace: kserve-ci-e2e-test resourceVersion: "25212" uid: 644ded6a-017b-4ee7-8ff4-8605510ae95d kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Created pod: error-404-isvc-a9ea4-predictor-756b95979d-vf9xt' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4-predictor-756b95979d.18c68125c2245351 namespace: kserve-ci-e2e-test resourceVersion: "25221" uid: 3704d66f-6d80-44a1-9b2d-931986bc4803 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25210" uid: a2ff034a-8968-46c8-af8b-8adc985247c4 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Scaled up replica set error-404-isvc-a9ea4-predictor-756b95979d from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4-predictor.18c68125c04f81ed namespace: kserve-ci-e2e-test resourceVersion: "25214" uid: 46d4a291-9859-418e-9c51-af977fab4ffd reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: dcaecdf0-24b8-42ad-9d2e-245e2a7cec51 kind: Event lastTimestamp: "2026-07-28T16:31:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:56Z" name: error-404-isvc-a9ea4-predictor.18c681293ebe4321 namespace: kserve-ci-e2e-test resourceVersion: "25403" uid: 7be2adfc-7bf4-4ffa-8d75-924bbc7a9ae6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25211" uid: dcaecdf0-24b8-42ad-9d2e-245e2a7cec51 kind: Event lastTimestamp: "2026-07-28T16:31:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:56Z" name: error-404-isvc-a9ea4-predictor.18c681293efdd8b0 namespace: kserve-ci-e2e-test resourceVersion: "25404" uid: 7d7ed7e2-27e9-43af-b746-871bc95aac61 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: dcaecdf0-24b8-42ad-9d2e-245e2a7cec51 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-a9ea4-predictor.18c6812cbddd1df8 namespace: kserve-ci-e2e-test resourceVersion: "26160" uid: cbc7b990-55e4-45fa-bffa-8363c64f913f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:32:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25402" uid: dcaecdf0-24b8-42ad-9d2e-245e2a7cec51 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-a9ea4-predictor.18c6812cbe584be5 namespace: kserve-ci-e2e-test resourceVersion: "26163" uid: aa5064b5-10b1-46c2-95d4-d0e7df73017a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 521d6e46-255e-4901-b5a0-adb9e91c37de kind: Event lastTimestamp: "2026-07-28T16:32:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:29Z" name: error-404-isvc-a9ea4.18c68125c08219ac namespace: kserve-ci-e2e-test resourceVersion: "25959" uid: ce6ac4d8-7455-437f-9bed-4b46514bfe12 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 521d6e46-255e-4901-b5a0-adb9e91c37de kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-a9ea4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9ea4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4.18c68125c0f58c61 namespace: kserve-ci-e2e-test resourceVersion: "25217" uid: d8139e11-8a12-446b-84c7-996339ae22b5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25179" uid: 521d6e46-255e-4901-b5a0-adb9e91c37de kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-a9ea4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-a9ea4.18c68125c0f5d7ea namespace: kserve-ci-e2e-test resourceVersion: "25218" uid: 3c35bddc-4b53-4608-bf28-d9f3d72c5950 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25953" uid: 521d6e46-255e-4901-b5a0-adb9e91c37de kind: Event lastTimestamp: "2026-07-28T16:32:29Z" message: InferenceService [error-404-isvc-a9ea4] is Ready metadata: creationTimestamp: "2026-07-28T16:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:29Z" name: error-404-isvc-a9ea4.18c68130f2e6f4c5 namespace: kserve-ci-e2e-test resourceVersion: "25955" uid: de311257-4dff-4f19-ab02-0aab3892ba69 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:23:26.641931Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21604" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b28c18afa0 namespace: kserve-ci-e2e-test resourceVersion: "21608" uid: 2df30112-0fd0-438c-ab50-c471dd316c03 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21615" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Add eth0 [10.134.0.46/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2a798968a namespace: kserve-ci-e2e-test resourceVersion: "21640" uid: 526f9918-0397-4956-a82b-88b3e5aab757 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2a9418f3e namespace: kserve-ci-e2e-test resourceVersion: "21642" uid: 79a72616-580d-49a3-904e-81c1fd97b8f8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2ac5cfd73 namespace: kserve-ci-e2e-test resourceVersion: "21644" uid: 3c0fd769-d3ed-4eb3-b2a3-66fb894d4207 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2ace38ab7 namespace: kserve-ci-e2e-test resourceVersion: "21645" uid: a8ca3c52-7fc3-445e-a7af-f0cf32f08848 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2acef6682 namespace: kserve-ci-e2e-test resourceVersion: "21646" uid: 12c0df81-3915-4c24-a38f-4ccc8fcb2f88 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2b06719ae namespace: kserve-ci-e2e-test resourceVersion: "21648" uid: bdbe00d0-9c36-4018-884e-154e480fd842 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b2b0f48ca1 namespace: kserve-ci-e2e-test resourceVersion: "21649" uid: 30ca2edf-97f0-4312-94a2-205860f25737 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:32:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.46:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:44Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c680b315e862b7 namespace: kserve-ci-e2e-test resourceVersion: "26250" uid: 9c45ff3f-544c-4110-a81a-1e7f5bb8897b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c68133b560c5ec namespace: kserve-ci-e2e-test resourceVersion: "26122" uid: 99685750-cd79-4b06-baa8-599947ae205d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c68133b56207cd namespace: kserve-ci-e2e-test resourceVersion: "26126" uid: 1ad7affd-4ed5-48cf-9530-b748a1771834 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk namespace: kserve-ci-e2e-test resourceVersion: "21606" uid: ec669095-1542-45fb-bfc9-cf8ca7b37332 kind: Event lastTimestamp: "2026-07-28T16:32:44Z" message: 'Readiness probe failed: Get "https://10.134.0.46:8643/healthz": dial tcp 10.134.0.46:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:44Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk.18c681348a61f967 namespace: kserve-ci-e2e-test resourceVersion: "26249" uid: ca54332a-6cb7-4fd6-857a-1dde7e023153 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-c3dc2-predictor-6987f9bc88 namespace: kserve-ci-e2e-test resourceVersion: "21593" uid: eb4c6f4c-f450-4c06-8fe7-bbf584dd0a70 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'Created pod: error-404-isvc-c3dc2-predictor-6987f9bc88-c96rk' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-c3dc2-predictor-6987f9bc88.18c680b28b61acdf namespace: kserve-ci-e2e-test resourceVersion: "21605" uid: f305802f-0a49-4116-a937-4c482fa928bb reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21591" uid: 0c29131f-0f32-4d18-bbb2-fad40cedbe99 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Scaled up replica set error-404-isvc-c3dc2-predictor-6987f9bc88 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-c3dc2-predictor.18c680b28938659b namespace: kserve-ci-e2e-test resourceVersion: "21594" uid: 95f29064-9974-4e07-935d-b7a55d1af651 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 08f9a95d-40f7-41e4-aeff-17fc555ca034 kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: error-404-isvc-c3dc2-predictor.18c680b607ecbe96 namespace: kserve-ci-e2e-test resourceVersion: "21912" uid: aadafcbe-54a3-4d0c-a972-50e196617bfa reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21592" uid: 08f9a95d-40f7-41e4-aeff-17fc555ca034 kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: error-404-isvc-c3dc2-predictor.18c680b608c36728 namespace: kserve-ci-e2e-test resourceVersion: "21913" uid: 63eee287-3274-4703-a833-a99375904694 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:24:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 08f9a95d-40f7-41e4-aeff-17fc555ca034 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: error-404-isvc-c3dc2-predictor.18c680bd070cf702 namespace: kserve-ci-e2e-test resourceVersion: "22298" uid: 953d35d7-cb62-4081-bf69-dc3eadfc36c0 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:24:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21785" uid: 08f9a95d-40f7-41e4-aeff-17fc555ca034 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: error-404-isvc-c3dc2-predictor.18c680bd081f1af6 namespace: kserve-ci-e2e-test resourceVersion: "22299" uid: 6f892905-dfce-4cf1-a748-31ec378eef4f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 72ea1f94-0705-496a-b6ce-9d520ea38657 kind: Event lastTimestamp: "2026-07-28T16:24:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:15Z" name: error-404-isvc-c3dc2.18c680b2893b616b namespace: kserve-ci-e2e-test resourceVersion: "22064" uid: 09bbfb56-1bdb-4f39-bb7c-9d64f76c143c reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 72ea1f94-0705-496a-b6ce-9d520ea38657 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'Failed to update status for InferenceService "error-404-isvc-c3dc2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-c3dc2.18c680b289fef5cd namespace: kserve-ci-e2e-test resourceVersion: "21600" uid: fa2f50d9-7468-41f2-9b47-77eee70bc3e8 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21568" uid: 72ea1f94-0705-496a-b6ce-9d520ea38657 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:26Z" name: error-404-isvc-c3dc2.18c680b289ff3c2c namespace: kserve-ci-e2e-test resourceVersion: "21601" uid: 911835a8-397a-43a3-b922-b43f81d8e43a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22062" uid: 72ea1f94-0705-496a-b6ce-9d520ea38657 kind: Event lastTimestamp: "2026-07-28T16:24:15Z" message: InferenceService [error-404-isvc-c3dc2] is Ready metadata: creationTimestamp: "2026-07-28T16:24:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:15Z" name: error-404-isvc-c3dc2.18c680bdd3c50502 namespace: kserve-ci-e2e-test resourceVersion: "22063" uid: cd2603ef-ca03-4c6f-bea3-0660be4fd2bd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:41.523265Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26167" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68133bd99e713 namespace: kserve-ci-e2e-test resourceVersion: "26171" uid: e62729e6-540a-43a6-bbc9-fcbfb040b106 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-d3a3d-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68133c944ae66 namespace: kserve-ci-e2e-test resourceVersion: "26185" uid: 3bdd56c9-e692-4bb1-a59d-20cf6cd7ee99 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26175" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Add eth0 [10.134.0.50/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68133fbb61998 namespace: kserve-ci-e2e-test resourceVersion: "26211" uid: c2561c2b-6334-4396-9950-a68fb0208f26 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68133fd28be5b namespace: kserve-ci-e2e-test resourceVersion: "26213" uid: 857760ea-ffed-45f3-b312-eea3136fb111 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c681340102530c namespace: kserve-ci-e2e-test resourceVersion: "26217" uid: 5184f7bd-8969-4d4d-a16c-f187fa58a4e8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c6813401919feb namespace: kserve-ci-e2e-test resourceVersion: "26218" uid: 4d89791e-10c6-4267-83e1-5d81bf915ac3 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68134019cad1c namespace: kserve-ci-e2e-test resourceVersion: "26219" uid: b6e18f38-bdfc-4376-a444-d6e6987fe8ac reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68134046cd2a1 namespace: kserve-ci-e2e-test resourceVersion: "26220" uid: bc306540-cbb3-4ded-88f0-8fb98c28c5b1 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c6813404ea0403 namespace: kserve-ci-e2e-test resourceVersion: "26221" uid: 348c475f-3659-4206-a97a-126cdf5bd2cc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:33:29Z" message: 'Readiness probe failed: dial tcp 10.134.0.50:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:29Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c6813441c03bf8 namespace: kserve-ci-e2e-test resourceVersion: "26793" uid: a9b00bd9-ea12-484a-be2d-ee1c03c8fb23 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:50:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:00Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68225bf0acd83 namespace: kserve-ci-e2e-test resourceVersion: "33808" uid: 78277246-4b0b-4d5e-9fa0-fef6e4b90b2a reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz namespace: kserve-ci-e2e-test resourceVersion: "26169" uid: 0756a0df-0845-4c28-ab0c-e213f648fa1a kind: Event lastTimestamp: "2026-07-28T16:50:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:00Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz.18c68225bf0b5ddf namespace: kserve-ci-e2e-test resourceVersion: "33809" uid: 73aed05a-a902-43cd-9fb5-eb95fbbedc83 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-d3a3d-predictor-5bd88cbf88 namespace: kserve-ci-e2e-test resourceVersion: "26154" uid: 042f48bd-3805-451e-a661-669199a81054 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Created pod: error-404-isvc-d3a3d-predictor-5bd88cbf88-j5dtz' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d-predictor-5bd88cbf88.18c68133bd3192aa namespace: kserve-ci-e2e-test resourceVersion: "26168" uid: 905bbeeb-ec80-405b-8c74-c76f81e2d268 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26152" uid: 0f7155ee-3d68-4ecc-a951-9ff9541a39f0 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Scaled up replica set error-404-isvc-d3a3d-predictor-5bd88cbf88 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d-predictor.18c68133bb7e7dd2 namespace: kserve-ci-e2e-test resourceVersion: "26155" uid: 01f600cf-824a-45bd-aff5-3a3ddcc61ae1 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: dfeb25a7-b095-4729-b217-00ea41687a9a kind: Event lastTimestamp: "2026-07-28T16:32:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:56Z" name: error-404-isvc-d3a3d-predictor.18c6813739eba992 namespace: kserve-ci-e2e-test resourceVersion: "26506" uid: d7112710-5099-4d60-8b0b-dc2362ce142e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26153" uid: dfeb25a7-b095-4729-b217-00ea41687a9a kind: Event lastTimestamp: "2026-07-28T16:32:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:56Z" name: error-404-isvc-d3a3d-predictor.18c681373a4e0386 namespace: kserve-ci-e2e-test resourceVersion: "26508" uid: cdee647b-e149-4174-9fd8-2db31b59f5db reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: dfeb25a7-b095-4729-b217-00ea41687a9a kind: Event lastTimestamp: "2026-07-28T16:34:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:11Z" name: error-404-isvc-d3a3d-predictor.18c6813ab9046938 namespace: kserve-ci-e2e-test resourceVersion: "27217" uid: 93a96780-a1a1-4c11-b589-5c287d4dac3a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:33:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26507" uid: dfeb25a7-b095-4729-b217-00ea41687a9a kind: Event lastTimestamp: "2026-07-28T16:34:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:11Z" name: error-404-isvc-d3a3d-predictor.18c6813ab9714651 namespace: kserve-ci-e2e-test resourceVersion: "27218" uid: 9594e22f-4737-4a6c-a116-34e877259041 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: af9488d7-680e-4d61-b4ae-de3012f22abf kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-d3a3d.18c68133bb874b7f namespace: kserve-ci-e2e-test resourceVersion: "26882" uid: 7a853961-0577-4e35-b380-bdbf4fda0c1f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: af9488d7-680e-4d61-b4ae-de3012f22abf kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Failed to update status for InferenceService "error-404-isvc-d3a3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d3a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d.18c68133bc13f56b namespace: kserve-ci-e2e-test resourceVersion: "26161" uid: 95e7cfa7-fc71-4477-b45d-473b36c87e4b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26137" uid: af9488d7-680e-4d61-b4ae-de3012f22abf kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-d3a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:41Z" name: error-404-isvc-d3a3d.18c68133bc14429a namespace: kserve-ci-e2e-test resourceVersion: "26165" uid: 52da3bd3-dac6-44b6-a59f-c7ec0a30370a reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:39Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26874" uid: af9488d7-680e-4d61-b4ae-de3012f22abf kind: Event lastTimestamp: "2026-07-28T16:33:39Z" message: InferenceService [error-404-isvc-d3a3d] is Ready metadata: creationTimestamp: "2026-07-28T16:33:39Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:39Z" name: error-404-isvc-d3a3d.18c6814151877644 namespace: kserve-ci-e2e-test resourceVersion: "26879" uid: 1d128814-2b01-46e2-a494-a70ede675729 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:20:56.222608Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19822" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808f8667e5f1 namespace: kserve-ci-e2e-test resourceVersion: "19828" uid: fdbd7f0c-d9e9-4d13-b23b-c82c208625af reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-daf0b-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808f9370bff6 namespace: kserve-ci-e2e-test resourceVersion: "19856" uid: 1f58cdee-99f8-4526-a775-97e2a5991d38 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19838" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: Add eth0 [10.134.0.40/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fc4c1381b namespace: kserve-ci-e2e-test resourceVersion: "19876" uid: 47109f7d-df81-40ac-963d-f3b290d10dbc reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fc64212a5 namespace: kserve-ci-e2e-test resourceVersion: "19878" uid: 41ced0be-7bc5-4e65-978a-5c6e5ceabb6a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fc9ec13cf namespace: kserve-ci-e2e-test resourceVersion: "19879" uid: d034da3d-0f77-4ec7-aa8b-e6c40cd29d50 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fca6f46c3 namespace: kserve-ci-e2e-test resourceVersion: "19880" uid: d780b290-e97b-4d98-9a83-297a50ec96d4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fca7a18a8 namespace: kserve-ci-e2e-test resourceVersion: "19881" uid: 87263afe-b116-4d85-b751-03e65e8c859c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fcddfae87 namespace: kserve-ci-e2e-test resourceVersion: "19882" uid: fb0c5fcd-59b7-4366-a27e-4a3f41e38ce0 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:20:57Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:57Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6808fce67f0fd namespace: kserve-ci-e2e-test resourceVersion: "19883" uid: 70cd8ca6-7ef0-4349-b9f5-43ec96a16b2b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:20:59Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:21:35Z" message: 'Readiness probe failed: dial tcp 10.134.0.40:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:20:59Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:21:35Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c6809033f79c78 namespace: kserve-ci-e2e-test resourceVersion: "20186" uid: b749103f-0661-43a5-af14-8dfd5d430b98 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c680a2312ea854 namespace: kserve-ci-e2e-test resourceVersion: "20765" uid: 51445993-7de3-4abb-8aa7-4f6b63e09ddf reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c680a2312f2ce2 namespace: kserve-ci-e2e-test resourceVersion: "20768" uid: dc2feaa5-530e-4c16-a891-399f06d6f232 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:20Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv namespace: kserve-ci-e2e-test resourceVersion: "19824" uid: 7eb2c05d-7e33-40f4-b59f-5fe0740fc3a0 kind: Event lastTimestamp: "2026-07-28T16:22:20Z" message: 'Readiness probe failed: Get "https://10.134.0.40:8643/healthz": dial tcp 10.134.0.40:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:20Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:20Z" name: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv.18c680a31055ca67 namespace: kserve-ci-e2e-test resourceVersion: "20863" uid: 066a8f95-01f7-4d4a-a5be-c8b9d358037a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-daf0b-predictor-788fff4fdd namespace: kserve-ci-e2e-test resourceVersion: "19814" uid: 132d6c5d-fc24-40c7-a6d7-3997090320f0 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Created pod: error-404-isvc-daf0b-predictor-788fff4fdd-nkfxv' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b-predictor-788fff4fdd.18c6808f85ed8ad9 namespace: kserve-ci-e2e-test resourceVersion: "19823" uid: d942f45e-d627-4904-93aa-1c37fe0ec13e reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19811" uid: bd933732-a795-4b28-a566-35b6b649b76c kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Scaled up replica set error-404-isvc-daf0b-predictor-788fff4fdd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b-predictor.18c6808f84306e8a namespace: kserve-ci-e2e-test resourceVersion: "19816" uid: 3e0e8471-8480-47e9-abe1-0ef19671e1e4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 3d05824d-7e45-4739-a7e7-6f105a7a02b7 kind: Event lastTimestamp: "2026-07-28T16:21:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:21:26Z" name: error-404-isvc-daf0b-predictor.18c6809302cdf6fd namespace: kserve-ci-e2e-test resourceVersion: "20116" uid: 267eae80-d6f4-4d64-b603-d0ebdf79f26b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19813" uid: 3d05824d-7e45-4739-a7e7-6f105a7a02b7 kind: Event lastTimestamp: "2026-07-28T16:21:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:21:26Z" name: error-404-isvc-daf0b-predictor.18c680930338696f namespace: kserve-ci-e2e-test resourceVersion: "20117" uid: bc521dbf-7fca-426c-9989-d02ae2d203f1 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 3d05824d-7e45-4739-a7e7-6f105a7a02b7 kind: Event lastTimestamp: "2026-07-28T16:22:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:11Z" name: error-404-isvc-daf0b-predictor.18c6809a01025bcc namespace: kserve-ci-e2e-test resourceVersion: "20654" uid: ec20e6f3-4bfa-4918-b03d-3f149e3a732c reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20007" uid: 3d05824d-7e45-4739-a7e7-6f105a7a02b7 kind: Event lastTimestamp: "2026-07-28T16:22:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:11Z" name: error-404-isvc-daf0b-predictor.18c6809a0195db19 namespace: kserve-ci-e2e-test resourceVersion: "20655" uid: 803aaa1a-1157-4018-b52d-9ae3a2de2141 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: ab31bf5a-f3f9-483f-ac9d-eb3899847d6b kind: Event lastTimestamp: "2026-07-28T16:21:45Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:21:45Z" name: error-404-isvc-daf0b.18c6808f84700310 namespace: kserve-ci-e2e-test resourceVersion: "20270" uid: bd7b16cc-abc0-421b-b71e-a5cc45ba275f reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: ab31bf5a-f3f9-483f-ac9d-eb3899847d6b kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Failed to update status for InferenceService "error-404-isvc-daf0b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b.18c6808f84eca312 namespace: kserve-ci-e2e-test resourceVersion: "19819" uid: b37d36f6-f062-4018-9cfd-2887ff451a84 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19779" uid: ab31bf5a-f3f9-483f-ac9d-eb3899847d6b kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:20:56Z" name: error-404-isvc-daf0b.18c6808f84ecf59f namespace: kserve-ci-e2e-test resourceVersion: "19821" uid: 4d5ca800-4452-4065-88ee-24b098361afd reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:21:45Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20267" uid: ab31bf5a-f3f9-483f-ac9d-eb3899847d6b kind: Event lastTimestamp: "2026-07-28T16:21:45Z" message: InferenceService [error-404-isvc-daf0b] is Ready metadata: creationTimestamp: "2026-07-28T16:21:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:21:45Z" name: error-404-isvc-daf0b.18c6809aefd2e41c namespace: kserve-ci-e2e-test resourceVersion: "20269" uid: d66acc3d-8143-46e8-8a61-f928cb3ba4e8 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:22:16.501998Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20789" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a2376e3d89 namespace: kserve-ci-e2e-test resourceVersion: "20793" uid: 24a5e032-e7bd-47b5-98de-247cac2a182a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "error-404-isvc-e5aa9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a241a6e8dc namespace: kserve-ci-e2e-test resourceVersion: "20809" uid: d4e3599f-e1ca-4c87-8251-aca7b3f1833f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20797" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Add eth0 [10.134.0.44/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a2765d2f8c namespace: kserve-ci-e2e-test resourceVersion: "20826" uid: 091b8627-5d8c-40bf-b8d2-5deb254260a4 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Container image "quay.io/opendatahub/error-404-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a277bb4152 namespace: kserve-ci-e2e-test resourceVersion: "20828" uid: 0003c50d-1d4a-4e90-bdb9-cb91d81c2e95 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a27ab7e8d3 namespace: kserve-ci-e2e-test resourceVersion: "20832" uid: 31b7fd87-035d-41e7-9290-d4ec82d1448a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a27b34e29c namespace: kserve-ci-e2e-test resourceVersion: "20833" uid: f953a800-7a76-4845-8b77-f9dc10f304c9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a27b40071b namespace: kserve-ci-e2e-test resourceVersion: "20834" uid: d4165f8e-5438-4935-9fbc-0132c0d2b40c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a27f631993 namespace: kserve-ci-e2e-test resourceVersion: "20835" uid: 0e4863a9-d4a0-4ede-8770-edd317998c82 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a27fe5b04f namespace: kserve-ci-e2e-test resourceVersion: "20836" uid: 5fcf641e-2917-4e2d-acdb-0fdc92b935a8 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T16:22:19Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:31:45Z" message: 'Readiness probe failed: dial tcp 10.134.0.44:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:19Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:45Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c680a2ef163931 namespace: kserve-ci-e2e-test resourceVersion: "25315" uid: 58a44b99-c20a-41a5-b360-d97bd3652aa7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c68125c5dcc4f3 namespace: kserve-ci-e2e-test resourceVersion: "25242" uid: ba5dda49-b687-4d4d-9122-201ab131ead9 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c68125c5dd4466 namespace: kserve-ci-e2e-test resourceVersion: "25244" uid: 047ea8aa-28e8-44c5-9e67-1352d7ca99b3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:45Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6 namespace: kserve-ci-e2e-test resourceVersion: "20791" uid: 9cec1195-36c6-4155-838e-f16d928c184b kind: Event lastTimestamp: "2026-07-28T16:31:45Z" message: 'Readiness probe failed: Get "https://10.134.0.44:8643/healthz": dial tcp 10.134.0.44:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:31:45Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:45Z" name: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6.18c68126b7d604a7 namespace: kserve-ci-e2e-test resourceVersion: "25314" uid: c2eeaf42-a3ce-406b-9609-30de82c23a3b reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: error-404-isvc-e5aa9-predictor-5f4f89b namespace: kserve-ci-e2e-test resourceVersion: "20782" uid: 1329a8c3-0d67-4eb9-a4c4-135153195a60 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'Created pod: error-404-isvc-e5aa9-predictor-5f4f89b-pj4n6' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9-predictor-5f4f89b.18c680a23701adb7 namespace: kserve-ci-e2e-test resourceVersion: "20790" uid: d1661d71-8455-4f4d-833c-e06474125126 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: error-404-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20780" uid: d50d613b-cc57-4b37-afc3-ece98e38cb79 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Scaled up replica set error-404-isvc-e5aa9-predictor-5f4f89b from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9-predictor.18c680a23549d1f7 namespace: kserve-ci-e2e-test resourceVersion: "20783" uid: 159d7bc1-62ab-41da-a40e-005710a901f3 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 2562c75b-8fae-4e0a-8277-edf9705bb82b kind: Event lastTimestamp: "2026-07-28T16:23:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:01Z" name: error-404-isvc-e5aa9-predictor.18c680a5b3edb4ed namespace: kserve-ci-e2e-test resourceVersion: "21219" uid: 127714e1-6d23-48fb-a710-16d89d603f47 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20781" uid: 2562c75b-8fae-4e0a-8277-edf9705bb82b kind: Event lastTimestamp: "2026-07-28T16:23:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:01Z" name: error-404-isvc-e5aa9-predictor.18c680a5b46bbfe7 namespace: kserve-ci-e2e-test resourceVersion: "21221" uid: b14b7cef-352c-46ea-801d-f0e6be9601f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 2562c75b-8fae-4e0a-8277-edf9705bb82b kind: Event lastTimestamp: "2026-07-28T16:23:31Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:31Z" name: error-404-isvc-e5aa9-predictor.18c680a9332f22e9 namespace: kserve-ci-e2e-test resourceVersion: "21700" uid: af7c579d-cee8-4a60-9b92-48d14761f56a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:46Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: error-404-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20975" uid: 2562c75b-8fae-4e0a-8277-edf9705bb82b kind: Event lastTimestamp: "2026-07-28T16:23:31Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:46Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:31Z" name: error-404-isvc-e5aa9-predictor.18c680a933958a38 namespace: kserve-ci-e2e-test resourceVersion: "21701" uid: 1cf799f9-5858-4f12-9051-d081b09cd7d4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: a37dc391-8a99-417f-bf0e-c6796549d30b kind: Event lastTimestamp: "2026-07-28T16:23:15Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:15Z" name: error-404-isvc-e5aa9.18c680a2355be91a namespace: kserve-ci-e2e-test resourceVersion: "21355" uid: 3ed6ec72-0c16-4122-8d1c-472fc8527614 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: a37dc391-8a99-417f-bf0e-c6796549d30b kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'Failed to update status for InferenceService "error-404-isvc-e5aa9": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e5aa9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9.18c680a235d513cd namespace: kserve-ci-e2e-test resourceVersion: "20786" uid: 9ffd4284-af48-4788-b9d9-9b608656b628 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "20762" uid: a37dc391-8a99-417f-bf0e-c6796549d30b kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "error-404-isvc-e5aa9": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:22:16Z" name: error-404-isvc-e5aa9.18c680a235d57cb2 namespace: kserve-ci-e2e-test resourceVersion: "20788" uid: 3c25d54c-bd07-485a-bf39-f16f8515497b reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:15Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: error-404-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21353" uid: a37dc391-8a99-417f-bf0e-c6796549d30b kind: Event lastTimestamp: "2026-07-28T16:23:15Z" message: InferenceService [error-404-isvc-e5aa9] is Ready metadata: creationTimestamp: "2026-07-28T16:23:15Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:15Z" name: error-404-isvc-e5aa9.18c680afff4c9aae namespace: kserve-ci-e2e-test resourceVersion: "21354" uid: 65c55313-97e6-4676-8dc8-3d20802ade16 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:10:07.188883Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14336" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff868fb402e namespace: kserve-ci-e2e-test resourceVersion: "14340" uid: 963e8746-5a00-4d47-85d9-8c8afd21af57 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14350" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Add eth0 [10.133.0.19/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff882dc2c4c namespace: kserve-ci-e2e-test resourceVersion: "14429" uid: acadc952-4c8b-4252-9605-6c99c901dacb reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0a55b7ab9c6c77d762de859d20496ef5a9b41cb492d5728752f25851b1e70916" already present on machine metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff884d8a77b namespace: kserve-ci-e2e-test resourceVersion: "14432" uid: 1b5ee598-2b58-46ec-8403-761acdfdfd1d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff887af1d74 namespace: kserve-ci-e2e-test resourceVersion: "14434" uid: a94c4fc6-5c4d-4a8e-b27e-34892a45693c reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff888206256 namespace: kserve-ci-e2e-test resourceVersion: "14436" uid: 2bcd65cf-781c-42ea-b25d-24c326630202 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T16:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:12Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c67ff9a781979f namespace: kserve-ci-e2e-test resourceVersion: "14518" uid: 59b65d86-3643-4192-9ad0-7d76dbe2143b reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:41Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 28.97s (28.97s including waiting). Image size: 1557910946 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:41Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c68000664517e7 namespace: kserve-ci-e2e-test resourceVersion: "14789" uid: d96eae39-4a97-40eb-a182-2e290a2fac39 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:41Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c680006da58940 namespace: kserve-ci-e2e-test resourceVersion: "14792" uid: bcd43d91-219a-4be2-9de5-4a903c8abe8e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:41Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c680006fee52fc namespace: kserve-ci-e2e-test resourceVersion: "14793" uid: ede9885f-8abd-486d-847f-23eb7819a1a0 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:41Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:10:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:41Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c680007002afe9 namespace: kserve-ci-e2e-test resourceVersion: "14794" uid: 18959161-74f8-4dae-a1bc-c861429db80e reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:42Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 1.261s (1.261s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:42Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c68000bb3564eb namespace: kserve-ci-e2e-test resourceVersion: "14807" uid: cdce806d-9622-4896-b5dc-c7be7444d75b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c68000bfbbc4c5 namespace: kserve-ci-e2e-test resourceVersion: "14811" uid: ee672c74-a3eb-4f57-9dde-e7aa8fd6c433 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c68000c0274662 namespace: kserve-ci-e2e-test resourceVersion: "14812" uid: 744a9406-71a5-49fb-8c56-8586c08a8fb9 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 10 eventTime: null firstTimestamp: "2026-07-28T16:10:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:11:59Z" message: 'Readiness probe failed: dial tcp 10.133.0.19:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:10:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:59Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c6800126f3eead namespace: kserve-ci-e2e-test resourceVersion: "15657" uid: ddbaf567-ea7e-44db-bb0c-9a8e4b6e1fac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c6801e9bee19e8 namespace: kserve-ci-e2e-test resourceVersion: "16232" uid: 4ceb7078-6d41-4ebf-bb2e-ff17a391aff7 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c6801e9beeb408 namespace: kserve-ci-e2e-test resourceVersion: "16236" uid: 49ecd347-e707-435d-8721-4bbba13aef28 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7 namespace: kserve-ci-e2e-test resourceVersion: "14337" uid: 3f237da2-64e5-4047-99e1-56bcce0f6e24 kind: Event lastTimestamp: "2026-07-28T16:12:55Z" message: 'Readiness probe failed: Get "https://10.133.0.19:8643/healthz": dial tcp 10.133.0.19:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:55Z" name: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7.18c6801fa75a9393 namespace: kserve-ci-e2e-test resourceVersion: "16354" uid: c42029bb-1d03-4c74-9a45-b8390e7569c7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-1-predictor-84c58bd689 namespace: kserve-ci-e2e-test resourceVersion: "14324" uid: 2631def2-b8ea-49ea-a30a-12ac1dc50377 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'Created pod: isvc-sklearn-graph-1-predictor-84c58bd689-44kr7' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor-84c58bd689.18c67ff868936664 namespace: kserve-ci-e2e-test resourceVersion: "14338" uid: cb8329f3-3bb8-44e7-8251-af324cfe7525 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14322" uid: 4132b6d5-fdff-413a-881e-ef3032ca4ee6 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Scaled up replica set isvc-sklearn-graph-1-predictor-84c58bd689 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-1-predictor.18c67ff860a1623a namespace: kserve-ci-e2e-test resourceVersion: "14325" uid: 624de8d4-92f7-42d1-aede-1feaba813d7d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 77f506d5-019a-4f2f-9aa4-02fb3727b5f4 kind: Event lastTimestamp: "2026-07-28T16:10:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:52Z" name: isvc-sklearn-graph-1-predictor.18c67ffbdfffe44d namespace: kserve-ci-e2e-test resourceVersion: "14893" uid: 5496a01d-4f51-4b9b-a86b-3bbb461f7438 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14323" uid: 77f506d5-019a-4f2f-9aa4-02fb3727b5f4 kind: Event lastTimestamp: "2026-07-28T16:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:52Z" name: isvc-sklearn-graph-1-predictor.18c67ffbe0751904 namespace: kserve-ci-e2e-test resourceVersion: "14894" uid: 4f2ce94f-86ea-4459-af06-51a35e80aecb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 77f506d5-019a-4f2f-9aa4-02fb3727b5f4 kind: Event lastTimestamp: "2026-07-28T16:12:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:22Z" name: isvc-sklearn-graph-1-predictor.18c680065d679d86 namespace: kserve-ci-e2e-test resourceVersion: "15848" uid: fbb71a05-f03a-4b48-8067-e1758cdf92bb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-1-predictor namespace: kserve-ci-e2e-test resourceVersion: "14605" uid: 77f506d5-019a-4f2f-9aa4-02fb3727b5f4 kind: Event lastTimestamp: "2026-07-28T16:12:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:22Z" name: isvc-sklearn-graph-1-predictor.18c680065dd6eb64 namespace: kserve-ci-e2e-test resourceVersion: "15849" uid: e95b669f-230f-4130-bb7f-6b3f4414bbb8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "14310" uid: b85dac2f-0192-475e-a52d-d6b74b70dfd1 kind: Event lastTimestamp: "2026-07-28T16:12:09Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:09Z" name: isvc-sklearn-graph-1.18c67ff860b0aa9e namespace: kserve-ci-e2e-test resourceVersion: "15758" uid: fd1eef46-1c17-4e17-bf49-291b94501ecd reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:09Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-1 namespace: kserve-ci-e2e-test resourceVersion: "15754" uid: b85dac2f-0192-475e-a52d-d6b74b70dfd1 kind: Event lastTimestamp: "2026-07-28T16:12:09Z" message: InferenceService [isvc-sklearn-graph-1] is Ready metadata: creationTimestamp: "2026-07-28T16:12:09Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:09Z" name: isvc-sklearn-graph-1.18c68014e79b0de3 namespace: kserve-ci-e2e-test resourceVersion: "15755" uid: 1dc7be52-7b64-4e10-9ab5-2a21456b7ebd reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:10:07.568888Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14416" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff87fa1a5ad namespace: kserve-ci-e2e-test resourceVersion: "14421" uid: 4469d80c-7a4c-445b-b232-9d6e5dba59a7 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-sklearn-graph-2-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff888ae58c8 namespace: kserve-ci-e2e-test resourceVersion: "14438" uid: 81428620-5725-42d4-988d-9867799091a5 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14425" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Add eth0 [10.133.0.21/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:10:08Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff8bd812648 namespace: kserve-ci-e2e-test resourceVersion: "14472" uid: ce04d847-dc55-41ca-8af2-c885634a49b8 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0a55b7ab9c6c77d762de859d20496ef5a9b41cb492d5728752f25851b1e70916" already present on machine metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff8bf368810 namespace: kserve-ci-e2e-test resourceVersion: "14474" uid: 36530874-59aa-44d3-8947-ff5c581c4b57 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff8c275a54e namespace: kserve-ci-e2e-test resourceVersion: "14477" uid: 521d8ebb-c43e-4779-814c-7148adb3c14f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff8c2f32880 namespace: kserve-ci-e2e-test resourceVersion: "14479" uid: 828432bf-62b6-4046-ae7b-374c7ae34daa reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:12Z" message: Pulling image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T16:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:12Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c67ff9a751ea11 namespace: kserve-ci-e2e-test resourceVersion: "14516" uid: 4880339c-e915-418e-8434-17536b5650b3 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: 'Successfully pulled image "quay.io/opendatahub/sklearn-serving-runtime:odh-pr-1821" in 27.723s (27.723s including waiting). Image size: 1557910946 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c680001bc041c4 namespace: kserve-ci-e2e-test resourceVersion: "14772" uid: 411235ea-1aa2-4538-9d7e-456284352de2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6800020f087fa namespace: kserve-ci-e2e-test resourceVersion: "14775" uid: f72ec57a-ed75-4238-9622-9df0494a8b7b reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6800022029ecd namespace: kserve-ci-e2e-test resourceVersion: "14776" uid: aae83997-bd9d-40a5-b464-240780f2d498 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c68000221230f0 namespace: kserve-ci-e2e-test resourceVersion: "14777" uid: 222a8e85-dbf7-49cf-8001-97d4b79245e5 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:42Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.568s (2.568s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:42Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c68000bb319e10 namespace: kserve-ci-e2e-test resourceVersion: "14806" uid: e926645a-efb2-4822-9c48-62e2a3db28b1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c68000bfba1c63 namespace: kserve-ci-e2e-test resourceVersion: "14810" uid: 1494331c-05c8-4e46-bdac-8a2be877e335 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c68000c0323460 namespace: kserve-ci-e2e-test resourceVersion: "14814" uid: 23be3fb8-6cef-4e9a-a7b7-a6e4c9402b99 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:10:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:11:50Z" message: 'Readiness probe failed: dial tcp 10.133.0.21:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:10:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:50Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6800126f318b1 namespace: kserve-ci-e2e-test resourceVersion: "15581" uid: 4ead6f4e-98ea-4785-9d84-86b9faf81b5c reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6801e97b3534d namespace: kserve-ci-e2e-test resourceVersion: "16199" uid: 761587c6-0f00-43eb-a5d8-c214db46328c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6801e97b41132 namespace: kserve-ci-e2e-test resourceVersion: "16201" uid: 50640597-e32a-4a54-86ac-cf7a5fdc7704 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk namespace: kserve-ci-e2e-test resourceVersion: "14419" uid: a85b88d2-526f-4050-a7a4-84425d6173fd kind: Event lastTimestamp: "2026-07-28T16:12:55Z" message: 'Readiness probe failed: Get "https://10.133.0.21:8643/healthz": dial tcp 10.133.0.21:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:12:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:55Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk.18c6801fa7592e27 namespace: kserve-ci-e2e-test resourceVersion: "16353" uid: 68d4e096-939c-4d5a-ad48-a63265821aac reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-sklearn-graph-2-predictor-5bc9545b47 namespace: kserve-ci-e2e-test resourceVersion: "14408" uid: 3c591f98-e23f-4840-8f6c-e52679ff7464 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'Created pod: isvc-sklearn-graph-2-predictor-5bc9545b47-wz2bk' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-2-predictor-5bc9545b47.18c67ff87f3a5654 namespace: kserve-ci-e2e-test resourceVersion: "14418" uid: af053518-a5c4-4850-9d7d-751e9d541b4a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14404" uid: bc1d2bd3-575f-4fb4-bf84-cda600367eae kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Scaled up replica set isvc-sklearn-graph-2-predictor-5bc9545b47 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-sklearn-graph-2-predictor.18c67ff8791c52d6 namespace: kserve-ci-e2e-test resourceVersion: "14409" uid: a0c699c0-bb7e-4bf9-8559-4933b7d59a67 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 8a9a0d2c-d704-4279-8dc6-9ea40f0c561d kind: Event lastTimestamp: "2026-07-28T16:10:52Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:52Z" name: isvc-sklearn-graph-2-predictor.18c67ffbf7ad34f9 namespace: kserve-ci-e2e-test resourceVersion: "14904" uid: b1b41134-d009-4dd7-a11a-e195851a0508 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14407" uid: 8a9a0d2c-d704-4279-8dc6-9ea40f0c561d kind: Event lastTimestamp: "2026-07-28T16:10:52Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:52Z" name: isvc-sklearn-graph-2-predictor.18c67ffbf815d8b8 namespace: kserve-ci-e2e-test resourceVersion: "14905" uid: f00c1ed4-95a4-4846-80d3-12bd43b08b8a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 8a9a0d2c-d704-4279-8dc6-9ea40f0c561d kind: Event lastTimestamp: "2026-07-28T16:12:22Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:22Z" name: isvc-sklearn-graph-2-predictor.18c6800674ca19ba namespace: kserve-ci-e2e-test resourceVersion: "15855" uid: 6972d738-a39b-42e7-b7ff-9d050dae0b82 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:07Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-sklearn-graph-2-predictor namespace: kserve-ci-e2e-test resourceVersion: "14617" uid: 8a9a0d2c-d704-4279-8dc6-9ea40f0c561d kind: Event lastTimestamp: "2026-07-28T16:12:22Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:22Z" name: isvc-sklearn-graph-2-predictor.18c68006753bdbb1 namespace: kserve-ci-e2e-test resourceVersion: "15856" uid: e036a36e-e851-4840-9d4a-05c3d5a00b75 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "14393" uid: 34c74845-f255-4701-a71f-c945af59bc62 kind: Event lastTimestamp: "2026-07-28T16:12:00Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:00Z" name: isvc-sklearn-graph-2.18c67ff8792d5059 namespace: kserve-ci-e2e-test resourceVersion: "15679" uid: 2db13a84-ccb0-4a77-932e-6c35a308fbce reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:00Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-sklearn-graph-2 namespace: kserve-ci-e2e-test resourceVersion: "15676" uid: 34c74845-f255-4701-a71f-c945af59bc62 kind: Event lastTimestamp: "2026-07-28T16:12:00Z" message: InferenceService [isvc-sklearn-graph-2] is Ready metadata: creationTimestamp: "2026-07-28T16:12:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:00Z" name: isvc-sklearn-graph-2.18c68012dfb62770 namespace: kserve-ci-e2e-test resourceVersion: "15678" uid: 07cac65a-e1d6-4067-abc4-babf260afe45 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:10:07.296092Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14363" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/isvc-xgboost-graph-predictor-555fc69d56-6xnd2 to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:10:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff86f5f0b6d namespace: kserve-ci-e2e-test resourceVersion: "14367" uid: 0caba915-fa48-4cd8-b814-5a07d396347a reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "isvc-xgboost-graph-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff87692aea8 namespace: kserve-ci-e2e-test resourceVersion: "14398" uid: bc3749c2-425d-4ea3-94ba-398db556e072 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14375" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Add eth0 [10.133.0.20/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:10:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff8ad0bec69 namespace: kserve-ci-e2e-test resourceVersion: "14457" uid: e2cc53f5-9de3-4c59-99a2-ef42584fe1a1 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Container image "quay.io/opendatahub/kserve-storage-initializer@sha256:0a55b7ab9c6c77d762de859d20496ef5a9b41cb492d5728752f25851b1e70916" already present on machine metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff8ae8108db namespace: kserve-ci-e2e-test resourceVersion: "14459" uid: d563e54a-3e32-419b-9ce1-7cd6bf15a527 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: 'Created container: storage-initializer' metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff8b287b347 namespace: kserve-ci-e2e-test resourceVersion: "14460" uid: a1c8073f-55bb-428d-be70-a6e9fabf62e4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:08Z" involvedObject: apiVersion: v1 fieldPath: spec.initContainers{storage-initializer} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:08Z" message: Started container storage-initializer metadata: creationTimestamp: "2026-07-28T16:10:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:08Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff8b3020a43 namespace: kserve-ci-e2e-test resourceVersion: "14462" uid: 1b9a4f8b-1553-499b-8d0c-b24c6f74ef25 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:12Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:12Z" message: Pulling image "kserve/xgbserver:latest" metadata: creationTimestamp: "2026-07-28T16:10:12Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:12Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c67ff9a76e27dc namespace: kserve-ci-e2e-test resourceVersion: "14517" uid: c5f916fc-21d7-42f5-9ba0-26cd44cae08f reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: 'Successfully pulled image "kserve/xgbserver:latest" in 28.171s (28.171s including waiting). Image size: 1307355896 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c6800036924232 namespace: kserve-ci-e2e-test resourceVersion: "14779" uid: dba6c25a-24a2-4c61-88b5-2d919eda3427 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c680003c23dcb1 namespace: kserve-ci-e2e-test resourceVersion: "14780" uid: 591e4997-11ae-40d4-986d-498e178ae7dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c680003d516ae0 namespace: kserve-ci-e2e-test resourceVersion: "14782" uid: 84e7ae2d-0fc0-4fe4-a432-12c653fdefbf reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:40Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:40Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:10:40Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:40Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c680003d6321ec namespace: kserve-ci-e2e-test resourceVersion: "14783" uid: ec2b22de-18fa-40fc-adfe-a89827bb0006 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:42Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.125s (2.125s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:42Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c68000bc16bc6e namespace: kserve-ci-e2e-test resourceVersion: "14808" uid: 56c6c0c8-0671-4c2a-b985-ae7aeb928087 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c68000bfb76e94 namespace: kserve-ci-e2e-test resourceVersion: "14809" uid: 8c8d347f-6bfa-4f62-9eac-70fd9151a3d3 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:10:43Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:43Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c68000c02c9841 namespace: kserve-ci-e2e-test resourceVersion: "14813" uid: 45419753-c29c-4818-bd30-08eab27ce658 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T16:10:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:11:39Z" message: 'Readiness probe failed: dial tcp 10.133.0.20:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:10:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:39Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c68000eae9a64f namespace: kserve-ci-e2e-test resourceVersion: "15340" uid: a74ff032-04b6-4b19-a282-5f2e297f8aec reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c6801ea052f51c namespace: kserve-ci-e2e-test resourceVersion: "16261" uid: 71b23ab5-7ee0-4c5a-ae2d-f7a3ba146ba6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c6801ea0539168 namespace: kserve-ci-e2e-test resourceVersion: "16264" uid: 57c53cee-a5d3-4412-8184-755eb7301ee6 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2 namespace: kserve-ci-e2e-test resourceVersion: "14364" uid: 89d15a04-9b9b-45c3-9dee-188d150229b7 kind: Event lastTimestamp: "2026-07-28T16:12:54Z" message: 'Readiness probe failed: Get "https://10.133.0.20:8643/healthz": dial tcp 10.133.0.20:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:12:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:54Z" name: isvc-xgboost-graph-predictor-555fc69d56-6xnd2.18c6801f6b8917c3 namespace: kserve-ci-e2e-test resourceVersion: "16344" uid: f52a7412-f207-4d49-a691-5c28be6834a4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: isvc-xgboost-graph-predictor-555fc69d56 namespace: kserve-ci-e2e-test resourceVersion: "14352" uid: aeb7a972-99d4-476c-8235-8a14308ab578 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: 'Created pod: isvc-xgboost-graph-predictor-555fc69d56-6xnd2' metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-xgboost-graph-predictor-555fc69d56.18c67ff86eef7f18 namespace: kserve-ci-e2e-test resourceVersion: "14365" uid: 1780b569-53f8-45a0-bbd2-898310ba4ea4 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14349" uid: 2f36348a-a062-4299-b070-570261249fc5 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Scaled up replica set isvc-xgboost-graph-predictor-555fc69d56 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:07Z" name: isvc-xgboost-graph-predictor.18c67ff86b9d11ff namespace: kserve-ci-e2e-test resourceVersion: "14354" uid: 0a9cdc7c-da5a-4914-b7f4-7d5900690150 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 8dc27ad9-0e0a-402f-a938-3dec561dcd0f kind: Event lastTimestamp: "2026-07-28T16:11:07Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:07Z" name: isvc-xgboost-graph-predictor.18c67ffbea52c9fc namespace: kserve-ci-e2e-test resourceVersion: "15021" uid: 5ca43b0a-3eaa-44e9-9e1a-7826c79858c4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14351" uid: 8dc27ad9-0e0a-402f-a938-3dec561dcd0f kind: Event lastTimestamp: "2026-07-28T16:11:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:07Z" name: isvc-xgboost-graph-predictor.18c67ffbeab719ef namespace: kserve-ci-e2e-test resourceVersion: "15022" uid: 0829b431-9949-4615-abd7-8560aba1ad8d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 8dc27ad9-0e0a-402f-a938-3dec561dcd0f kind: Event lastTimestamp: "2026-07-28T16:12:07Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:07Z" name: isvc-xgboost-graph-predictor.18c68009e638cb82 namespace: kserve-ci-e2e-test resourceVersion: "15728" uid: d0302633-2358-48ca-b17d-226f561adfe4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:11:22Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: isvc-xgboost-graph-predictor namespace: kserve-ci-e2e-test resourceVersion: "14610" uid: 8dc27ad9-0e0a-402f-a938-3dec561dcd0f kind: Event lastTimestamp: "2026-07-28T16:12:07Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:11:22Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:07Z" name: isvc-xgboost-graph-predictor.18c68009e6a09fde namespace: kserve-ci-e2e-test resourceVersion: "15729" uid: 5016f785-e859-425a-8af2-66667c59be10 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 9 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "14335" uid: 19d915b1-d2e4-4297-ab77-dbc9a06651ea kind: Event lastTimestamp: "2026-07-28T16:11:49Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:49Z" name: isvc-xgboost-graph.18c67ff86bcaadd7 namespace: kserve-ci-e2e-test resourceVersion: "15572" uid: da64c659-3fca-42d3-860c-5bb28375c790 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:49Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: isvc-xgboost-graph namespace: kserve-ci-e2e-test resourceVersion: "15570" uid: 19d915b1-d2e4-4297-ab77-dbc9a06651ea kind: Event lastTimestamp: "2026-07-28T16:11:49Z" message: InferenceService [isvc-xgboost-graph] is Ready metadata: creationTimestamp: "2026-07-28T16:11:49Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:49Z" name: isvc-xgboost-graph.18c680104f21fab4 namespace: kserve-ci-e2e-test resourceVersion: "15571" uid: eec1d0c4-10e1-49c9-8b5b-1b5b7553a6f7 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:12:36.947535Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15991" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/model-chainer-fff6dfdd4-k75lz to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:12:36Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:12:36Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b474ad48f namespace: kserve-ci-e2e-test resourceVersion: "15997" uid: 22c9b329-3e92-4477-b3f0-7e3d16e8bd85 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:37Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "model-chainer-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:37Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b520550f3 namespace: kserve-ci-e2e-test resourceVersion: "16006" uid: fbc8b601-a34d-4958-bb1f-933e4fa9f408 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:37Z" involvedObject: apiVersion: v1 kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "16000" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:37Z" message: Add eth0 [10.134.0.35/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:12:37Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b85528bc0 namespace: kserve-ci-e2e-test resourceVersion: "16017" uid: 1ef3beac-a9dc-4b99-b321-4c5c6100aca9 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:38Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:38Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b8775cacc namespace: kserve-ci-e2e-test resourceVersion: "16019" uid: 2a55a96a-abd3-41b1-90c0-16cd93f79016 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:38Z" message: 'Created container: model-chainer' metadata: creationTimestamp: "2026-07-28T16:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:38Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b8af25d58 namespace: kserve-ci-e2e-test resourceVersion: "16021" uid: dce47b7f-3f28-4e41-802e-8fba27d4f64f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:38Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:38Z" message: Started container model-chainer metadata: creationTimestamp: "2026-07-28T16:12:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:38Z" name: model-chainer-fff6dfdd4-k75lz.18c6801b8b835599 namespace: kserve-ci-e2e-test resourceVersion: "16022" uid: 9e65cfe9-cedc-4762-920e-1a643b72cd4f reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Stopping container model-chainer metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: model-chainer-fff6dfdd4-k75lz.18c6801e900bf024 namespace: kserve-ci-e2e-test resourceVersion: "16176" uid: bd6073b0-55b2-457e-837d-cc28ff113190 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:12:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{model-chainer} kind: Pod name: model-chainer-fff6dfdd4-k75lz namespace: kserve-ci-e2e-test resourceVersion: "15993" uid: 58640b38-9c01-4db7-aaee-5a5b1b665374 kind: Event lastTimestamp: "2026-07-28T16:13:19Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:12:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:13:19Z" name: model-chainer-fff6dfdd4-k75lz.18c6801f6a16291b namespace: kserve-ci-e2e-test resourceVersion: "16556" uid: f6013968-e14d-4a03-b7dd-aebcdf7404b8 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:36Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: model-chainer-fff6dfdd4 namespace: kserve-ci-e2e-test resourceVersion: "15988" uid: 75c04f57-69e3-4b70-a99f-32b7348d869e kind: Event lastTimestamp: "2026-07-28T16:12:36Z" message: 'Created pod: model-chainer-fff6dfdd4-k75lz' metadata: creationTimestamp: "2026-07-28T16:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:36Z" name: model-chainer-fff6dfdd4.18c6801b46babb2f namespace: kserve-ci-e2e-test resourceVersion: "15992" uid: 7d709971-7616-48ed-9a6c-4fcabe7d65b1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:36Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "15986" uid: fdb5dc48-77c9-464a-b45f-fd7b63ee7003 kind: Event lastTimestamp: "2026-07-28T16:12:36Z" message: Scaled up replica set model-chainer-fff6dfdd4 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:12:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:36Z" name: model-chainer.18c6801b4589968a namespace: kserve-ci-e2e-test resourceVersion: "15989" uid: 2c3e0a6d-9d99-4fa9-96c0-8bec1316a3e9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:37Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: model-chainer namespace: kserve-ci-e2e-test resourceVersion: "16002" uid: 24eaa566-efb3-4aa5-a33a-8beeabd60af4 kind: Event lastTimestamp: "2026-07-28T16:12:37Z" message: InferenceGraph [model-chainer] is Ready metadata: creationTimestamp: "2026-07-28T16:12:37Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:37Z" name: model-chainer.18c6801b4c5999ff namespace: kserve-ci-e2e-test resourceVersion: "16004" uid: 467120ff-e008-4990-9a5f-6251bdcea3a2 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:13:51.315531Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16819" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-0402e-5f87ffd8fd-xs84s to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802c97f88ca9 namespace: kserve-ci-e2e-test resourceVersion: "16825" uid: b402de2f-c4f6-429f-98db-826c509806c9 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:51Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16829" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: Add eth0 [10.133.0.22/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802cb1688a5c namespace: kserve-ci-e2e-test resourceVersion: "16841" uid: b79e9b2d-4a3e-4b5d-99c9-81d27db6bf5a reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802cb30313f6 namespace: kserve-ci-e2e-test resourceVersion: "16843" uid: d1dc96c7-9bc3-4467-89ac-366823900357 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:13:53Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" in 2.066s (2.066s including waiting). Image size: 218225243 bytes.' metadata: creationTimestamp: "2026-07-28T16:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:13:53Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802d2e30781f namespace: kserve-ci-e2e-test resourceVersion: "16856" uid: e2da0afd-eb02-44b2-894e-775dc1d1fba6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:13:53Z" message: 'Created container: sequence-graph-0402e' metadata: creationTimestamp: "2026-07-28T16:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:13:53Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802d30cec99e namespace: kserve-ci-e2e-test resourceVersion: "16857" uid: b9e321ee-496c-4d13-b645-cd013a5e98a8 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:13:53Z" message: Started container sequence-graph-0402e metadata: creationTimestamp: "2026-07-28T16:13:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:13:53Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6802d315999ad namespace: kserve-ci-e2e-test resourceVersion: "16859" uid: e2735966-4ab5-4d50-9012-a4237802780d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:05Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:22:05Z" message: Stopping container sequence-graph-0402e metadata: creationTimestamp: "2026-07-28T16:22:05Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:05Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c6809fc0340dce namespace: kserve-ci-e2e-test resourceVersion: "20442" uid: 61ff3df1-16bc-47da-bb0b-14611e4e27a2 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:10Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-0402e} kind: Pod name: sequence-graph-0402e-5f87ffd8fd-xs84s namespace: kserve-ci-e2e-test resourceVersion: "16823" uid: 67851d84-2145-4769-ab2e-79d73df863a6 kind: Event lastTimestamp: "2026-07-28T16:22:35Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:22:10Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:35Z" name: sequence-graph-0402e-5f87ffd8fd-xs84s.18c680a0d0975d30 namespace: kserve-ci-e2e-test resourceVersion: "21007" uid: 3d8355f0-391e-448d-8ed7-5542274abe54 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-0402e-5f87ffd8fd namespace: kserve-ci-e2e-test resourceVersion: "16815" uid: da31a93a-55e5-4660-ac1a-0a1a2079a578 kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: 'Created pod: sequence-graph-0402e-5f87ffd8fd-xs84s' metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e-5f87ffd8fd.18c6802c97733ec9 namespace: kserve-ci-e2e-test resourceVersion: "16822" uid: eece3695-778f-469c-bcc6-936cb884ab82 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-0402e namespace: kserve-ci-e2e-test resourceVersion: "16813" uid: fc1e9ce6-5f42-4880-99a5-426ae0c43981 kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: Scaled up replica set sequence-graph-0402e-5f87ffd8fd from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e.18c6802c95c5926d namespace: kserve-ci-e2e-test resourceVersion: "16816" uid: 5fe43cae-acb1-4881-a1c6-6886868b632c reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:51Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-0402e namespace: kserve-ci-e2e-test resourceVersion: "16833" uid: 1fd2a940-aac2-47ae-abd1-b22f6d1c788c kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: InferenceGraph [sequence-graph-0402e] is Ready metadata: creationTimestamp: "2026-07-28T16:13:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:13:51Z" name: sequence-graph-0402e.18c6802c9d7521ae namespace: kserve-ci-e2e-test resourceVersion: "16835" uid: ae4d7d34-0715-4da6-b4b5-cf4723c3fb53 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0402e namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: 7fe8f8dd-0cf8-48d4-9fa2-2119a601952b kind: Event lastTimestamp: "2026-07-28T16:14:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:14:21Z" name: sequence-graph-0402e.18c6803014802d65 namespace: kserve-ci-e2e-test resourceVersion: "17061" uid: 2a4d3e04-e56b-4b9a-b8eb-ff02191bf4a2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:14:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-0402e namespace: kserve-ci-e2e-test resourceVersion: "16814" uid: 7fe8f8dd-0cf8-48d4-9fa2-2119a601952b kind: Event lastTimestamp: "2026-07-28T16:14:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:14:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:14:21Z" name: sequence-graph-0402e.18c6803014dbb75b namespace: kserve-ci-e2e-test resourceVersion: "17062" uid: 9f9ca900-1880-472d-982f-dcd6b08c224a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:23:16.144005Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21373" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-7c1b1-5f49957547-pnznt to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b01a5f1155 namespace: kserve-ci-e2e-test resourceVersion: "21379" uid: a7436490-48c3-4eb5-838c-ad2b407ca360 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "sequence-graph-7c1b1-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b023bab819 namespace: kserve-ci-e2e-test resourceVersion: "21389" uid: 190edefe-212d-4076-8759-bd59b696ba05 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:17Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21382" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:17Z" message: Add eth0 [10.133.0.24/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:23:17Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b057f63cb0 namespace: kserve-ci-e2e-test resourceVersion: "21405" uid: 4537c3bf-adb8-4725-bfa3-4fe61b4f1b26 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c1b1} kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:17Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:17Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b059a7feb1 namespace: kserve-ci-e2e-test resourceVersion: "21407" uid: 24406b0f-1a47-4b5c-b533-b5f1620efe06 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c1b1} kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:17Z" message: 'Created container: sequence-graph-7c1b1' metadata: creationTimestamp: "2026-07-28T16:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:17Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b05c6fa912 namespace: kserve-ci-e2e-test resourceVersion: "21409" uid: f28b2111-a95a-41a4-b7dc-fc7f8a2439aa reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c1b1} kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:17Z" message: Started container sequence-graph-7c1b1 metadata: creationTimestamp: "2026-07-28T16:23:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:17Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b05cebbe98 namespace: kserve-ci-e2e-test resourceVersion: "21410" uid: a9ed4156-32bc-42be-b1f8-cb6ebff9cfda reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c1b1} kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Stopping container sequence-graph-7c1b1 metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b27210843e namespace: kserve-ci-e2e-test resourceVersion: "21494" uid: d198c7b1-33b3-413a-a7f9-5aa8486f6e62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:23:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-7c1b1} kind: Pod name: sequence-graph-7c1b1-5f49957547-pnznt namespace: kserve-ci-e2e-test resourceVersion: "21375" uid: 00f7340e-bc02-4848-8e61-5e66fa3d99f1 kind: Event lastTimestamp: "2026-07-28T16:23:53Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:23:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:53Z" name: sequence-graph-7c1b1-5f49957547-pnznt.18c680b2fc8ec864 namespace: kserve-ci-e2e-test resourceVersion: "21882" uid: 826dadd8-6fbd-4f54-bfdd-dd728e7948cb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-7c1b1-5f49957547 namespace: kserve-ci-e2e-test resourceVersion: "21370" uid: cfc08745-13f8-4f10-84bc-87bc590f7e90 kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: 'Created pod: sequence-graph-7c1b1-5f49957547-pnznt' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1-5f49957547.18c680b019d3261b namespace: kserve-ci-e2e-test resourceVersion: "21376" uid: 9cbdb861-2f14-405c-8a6b-31c70ae2a700 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21368" uid: 3071625b-2dfe-400b-b1c0-5b45e5d515ec kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: Scaled up replica set sequence-graph-7c1b1-5f49957547 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1.18c680b0188dda69 namespace: kserve-ci-e2e-test resourceVersion: "21371" uid: 6bd1127a-154c-4dd3-8f52-564b64eb65c5 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21384" uid: 33c94c81-b5d9-4ae1-a5b4-21fed1f60ed6 kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: InferenceGraph [sequence-graph-7c1b1] is Ready metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1.18c680b01d4a5e17 namespace: kserve-ci-e2e-test resourceVersion: "21386" uid: 95511bc1-750c-415a-917f-12c417cb3a50 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 33c94c81-b5d9-4ae1-a5b4-21fed1f60ed6 kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: 'Failed to update status for InferenceGraph "sequence-graph-7c1b1": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-7c1b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1.18c680b020ced5e4 namespace: kserve-ci-e2e-test resourceVersion: "21387" uid: eaf7a37d-3417-47ac-9f0e-13d2607fd7e9 reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21361" uid: 33c94c81-b5d9-4ae1-a5b4-21fed1f60ed6 kind: Event lastTimestamp: "2026-07-28T16:23:16Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-7c1b1": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:16Z" name: sequence-graph-7c1b1.18c680b020cf2b19 namespace: kserve-ci-e2e-test resourceVersion: "21388" uid: 473bf004-eb55-4cce-a7fd-c1581e8849cc reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:24:26.500796Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22152" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/sequence-graph-c3dc2-7bdd665dff-5qxg6 to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c680c07bf67787 namespace: kserve-ci-e2e-test resourceVersion: "22160" uid: bb115f85-5f27-4755-abd0-8b70f19b836c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: v1 kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22165" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: Add eth0 [10.133.0.26/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c680c095b3e6da namespace: kserve-ci-e2e-test resourceVersion: "22177" uid: 755c0e91-7cb4-46e2-8383-ca48e26cf761 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3dc2} kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c680c096ffdd5e namespace: kserve-ci-e2e-test resourceVersion: "22179" uid: 1fd7b40d-4141-4e4e-b0c9-312c84619e1b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3dc2} kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:24:27Z" message: 'Created container: sequence-graph-c3dc2' metadata: creationTimestamp: "2026-07-28T16:24:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:24:27Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c680c09a0c6113 namespace: kserve-ci-e2e-test resourceVersion: "22181" uid: d77d7577-5096-4404-b78b-11dfbeade39a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3dc2} kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:24:27Z" message: Started container sequence-graph-c3dc2 metadata: creationTimestamp: "2026-07-28T16:24:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:24:27Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c680c09a86db35 namespace: kserve-ci-e2e-test resourceVersion: "22182" uid: d634ca44-3dc8-4307-96c1-8dc11aebec2c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3dc2} kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Stopping container sequence-graph-c3dc2 metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c68133aa103806 namespace: kserve-ci-e2e-test resourceVersion: "26055" uid: 6e28cc78-3846-4d97-8abd-cb2a4bd53859 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{sequence-graph-c3dc2} kind: Pod name: sequence-graph-c3dc2-7bdd665dff-5qxg6 namespace: kserve-ci-e2e-test resourceVersion: "22154" uid: 365cb4ed-4f70-41dd-8170-92d3d598570c kind: Event lastTimestamp: "2026-07-28T16:33:08Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:08Z" name: sequence-graph-c3dc2-7bdd665dff-5qxg6.18c6813444c4d920 namespace: kserve-ci-e2e-test resourceVersion: "26609" uid: 93f2699a-b286-44de-859b-b292a529c3c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: sequence-graph-c3dc2-7bdd665dff namespace: kserve-ci-e2e-test resourceVersion: "22149" uid: 82b8c0ac-5359-4417-a651-e3e8a18c2082 kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: 'Created pod: sequence-graph-c3dc2-7bdd665dff-5qxg6' metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2-7bdd665dff.18c680c07b72f96c namespace: kserve-ci-e2e-test resourceVersion: "22155" uid: 036781da-85b9-4a77-b49c-9d33f30ff683 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22147" uid: e557cdbc-d917-479a-bfca-b6d92d52c6af kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: Scaled up replica set sequence-graph-c3dc2-7bdd665dff from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2.18c680c07a5aab80 namespace: kserve-ci-e2e-test resourceVersion: "22150" uid: f1468356-9eab-452c-9e8f-3879c7a057ca reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22167" uid: d5c64b5d-fb4f-417c-a86b-a7e9a2473937 kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: InferenceGraph [sequence-graph-c3dc2] is Ready metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2.18c680c07ec20f9d namespace: kserve-ci-e2e-test resourceVersion: "22168" uid: 3ec82d39-591e-46db-a21a-070854cac870 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22141" uid: d5c64b5d-fb4f-417c-a86b-a7e9a2473937 kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: 'Failed to update status for InferenceGraph "sequence-graph-c3dc2": Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2.18c680c08187c639 namespace: kserve-ci-e2e-test resourceVersion: "22170" uid: 924ebe6a-51fd-4f34-9bc9-369baf4fe62c reason: UpdateFailed reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22141" uid: d5c64b5d-fb4f-417c-a86b-a7e9a2473937 kind: Event lastTimestamp: "2026-07-28T16:24:26Z" message: 'fails to update InferenceGraph status: Operation cannot be fulfilled on inferencegraphs.serving.kserve.io "sequence-graph-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:24:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:26Z" name: sequence-graph-c3dc2.18c680c0818825f4 namespace: kserve-ci-e2e-test resourceVersion: "22171" uid: 6a7aa031-b663-419b-9b33-f80452372947 reason: InternalError reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22148" uid: 63b01ee8-0b43-45ed-a006-cd88b2cefdf0 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: sequence-graph-c3dc2.18c680c3f905e5f5 namespace: kserve-ci-e2e-test resourceVersion: "22291" uid: 5f2dedbe-9606-4a5d-a877-f76b7767869f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22148" uid: 63b01ee8-0b43-45ed-a006-cd88b2cefdf0 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:24:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: sequence-graph-c3dc2.18c680c3f96e62f3 namespace: kserve-ci-e2e-test resourceVersion: "22293" uid: 969ff908-98f9-4902-b6c8-30927a24f1b8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: 63b01ee8-0b43-45ed-a006-cd88b2cefdf0 kind: Event lastTimestamp: "2026-07-28T16:24:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:56Z" name: sequence-graph-c3dc2.18c680c77847ab4d namespace: kserve-ci-e2e-test resourceVersion: "22399" uid: 53851dc1-0574-4128-8db3-f3009b79e8c2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: sequence-graph-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22292" uid: 63b01ee8-0b43-45ed-a006-cd88b2cefdf0 kind: Event lastTimestamp: "2026-07-28T16:24:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:56Z" name: sequence-graph-c3dc2.18c680c778985994 namespace: kserve-ci-e2e-test resourceVersion: "22401" uid: 3bc9d920-c4d7-4ee1-8fc0-9d17ad19e03b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:34:01.697577Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27111" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-49c25-56ffcb7444-z2gf6 to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:34:01Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:34:01Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c68146685cc1ef namespace: kserve-ci-e2e-test resourceVersion: "27116" uid: a1cfa3cc-5dc0-489c-b775-134f6de7706d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:02Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27120" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:34:02Z" message: Add eth0 [10.133.0.29/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:34:02Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c68146821ebe9f namespace: kserve-ci-e2e-test resourceVersion: "27132" uid: 1fba7598-5c77-4609-962c-1e0a4e7ab9de reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-49c25} kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:34:02Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:02Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c68146837aefbf namespace: kserve-ci-e2e-test resourceVersion: "27134" uid: 7a0c5035-dbdf-4b4b-9633-3bb6ed5b64c6 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-49c25} kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:34:02Z" message: 'Created container: splitter-graph-49c25' metadata: creationTimestamp: "2026-07-28T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:02Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c68146864cc792 namespace: kserve-ci-e2e-test resourceVersion: "27135" uid: 3d887f75-4fa7-4a58-8c2e-f32998db43e5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:02Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-49c25} kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:34:02Z" message: Started container splitter-graph-49c25 metadata: creationTimestamp: "2026-07-28T16:34:02Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:34:02Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c6814686cc4b5a namespace: kserve-ci-e2e-test resourceVersion: "27136" uid: 8b386e9b-50c0-4a73-a7f0-2e48acd1e26e reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-49c25} kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:42:16Z" message: Stopping container splitter-graph-49c25 metadata: creationTimestamp: "2026-07-28T16:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:16Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c681b995350b6b namespace: kserve-ci-e2e-test resourceVersion: "30563" uid: 3f159dfb-bd38-435d-9ca3-dad7f94fb194 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-49c25} kind: Pod name: splitter-graph-49c25-56ffcb7444-z2gf6 namespace: kserve-ci-e2e-test resourceVersion: "27113" uid: 6722563b-adb3-47fb-bb40-41e7567baf9a kind: Event lastTimestamp: "2026-07-28T16:42:43Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:43Z" name: splitter-graph-49c25-56ffcb7444-z2gf6.18c681ba2d4e3089 namespace: kserve-ci-e2e-test resourceVersion: "30806" uid: 73cd941e-a342-4c5d-ad9d-8caae44ce119 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:01Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-49c25-56ffcb7444 namespace: kserve-ci-e2e-test resourceVersion: "27108" uid: ad683814-a699-456c-9303-37f6aceced81 kind: Event lastTimestamp: "2026-07-28T16:34:01Z" message: 'Created pod: splitter-graph-49c25-56ffcb7444-z2gf6' metadata: creationTimestamp: "2026-07-28T16:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:01Z" name: splitter-graph-49c25-56ffcb7444.18c6814667e2c6bc namespace: kserve-ci-e2e-test resourceVersion: "27112" uid: 5d3854f4-a3b2-40b8-a68f-0884d4158913 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:01Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-49c25 namespace: kserve-ci-e2e-test resourceVersion: "27105" uid: fa5e3727-2996-416c-98e6-c6f6b9252ca9 kind: Event lastTimestamp: "2026-07-28T16:34:01Z" message: Scaled up replica set splitter-graph-49c25-56ffcb7444 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:01Z" name: splitter-graph-49c25.18c68146672dc404 namespace: kserve-ci-e2e-test resourceVersion: "27109" uid: cdf6b60b-3fd8-4e11-b36e-4845f52d75d4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:01Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-49c25 namespace: kserve-ci-e2e-test resourceVersion: "27122" uid: fdef7ae6-832f-4a3a-93bd-8b833f35766c kind: Event lastTimestamp: "2026-07-28T16:34:01Z" message: InferenceGraph [splitter-graph-49c25] is Ready metadata: creationTimestamp: "2026-07-28T16:34:01Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:34:01Z" name: splitter-graph-49c25.18c681466c9a19f4 namespace: kserve-ci-e2e-test resourceVersion: "27124" uid: 3f67b14a-c367-4ff1-8457-3145fd1ad859 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-49c25 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 68fd659f-1ce3-4f03-96f0-764653e938a6 kind: Event lastTimestamp: "2026-07-28T16:34:16Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:16Z" name: splitter-graph-49c25.18c68149e5cca99a namespace: kserve-ci-e2e-test resourceVersion: "27256" uid: 8b6b97cc-0fa6-4c35-8304-ab01493d2aa6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: splitter-graph-49c25 namespace: kserve-ci-e2e-test resourceVersion: "27107" uid: 68fd659f-1ce3-4f03-96f0-764653e938a6 kind: Event lastTimestamp: "2026-07-28T16:34:16Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:16Z" name: splitter-graph-49c25.18c68149e657a011 namespace: kserve-ci-e2e-test resourceVersion: "27258" uid: 9eeba3c1-7107-4c47-b458-9501c740339d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:41.389686Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26112" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/splitter-graph-a9ea4-7dbf9cdd7d-smspf to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:41Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133b5a3a4fe namespace: kserve-ci-e2e-test resourceVersion: "26127" uid: a097f9d5-081b-4453-abaf-7c501411db66 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "splitter-graph-a9ea4-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133c3444004 namespace: kserve-ci-e2e-test resourceVersion: "26179" uid: 4b8a2bbc-9bab-4c14-b038-4312c44b429f reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26134" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Add eth0 [10.133.0.27/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:42Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133f391a1e6 namespace: kserve-ci-e2e-test resourceVersion: "26205" uid: eb0bb1f2-33a1-45a0-8fb0-c76a134b2508 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a9ea4} kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133f4ec2fae namespace: kserve-ci-e2e-test resourceVersion: "26207" uid: 9c13a1e6-b3a3-43cc-b6d8-e0b0dda0880d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a9ea4} kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: 'Created container: splitter-graph-a9ea4' metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133f8fd3805 namespace: kserve-ci-e2e-test resourceVersion: "26209" uid: 5b1b6f09-782f-45df-9ee4-8cb3e3475d46 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a9ea4} kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:42Z" message: Started container splitter-graph-a9ea4 metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:42Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c68133f979cc71 namespace: kserve-ci-e2e-test resourceVersion: "26210" uid: 4fdb0063-be44-406c-a1b2-4c9f81104d31 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a9ea4} kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Stopping container splitter-graph-a9ea4 metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:51Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c681360cb607fe namespace: kserve-ci-e2e-test resourceVersion: "26332" uid: 2c3d74d5-bdf1-4ccf-962a-1babe5bcc2da reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:32:53Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{splitter-graph-a9ea4} kind: Pod name: splitter-graph-a9ea4-7dbf9cdd7d-smspf namespace: kserve-ci-e2e-test resourceVersion: "26120" uid: 80ce2218-d7c9-40de-a294-d7db4fbbb4bc kind: Event lastTimestamp: "2026-07-28T16:33:18Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:32:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:18Z" name: splitter-graph-a9ea4-7dbf9cdd7d-smspf.18c681368eaf23da namespace: kserve-ci-e2e-test resourceVersion: "26693" uid: 33cb8fd9-661a-441e-a519-d1d98aed4ac4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: splitter-graph-a9ea4-7dbf9cdd7d namespace: kserve-ci-e2e-test resourceVersion: "26092" uid: 806a8804-e1e5-4dd0-812b-c479e5049d4c kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Created pod: splitter-graph-a9ea4-7dbf9cdd7d-smspf' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: splitter-graph-a9ea4-7dbf9cdd7d.18c68133b4dd8f21 namespace: kserve-ci-e2e-test resourceVersion: "26117" uid: 381d51e2-d6a5-46f5-b8a5-40fb64209c77 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: splitter-graph-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "26086" uid: c40e8dc1-7da7-4838-8925-e4a89c3ab977 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Scaled up replica set splitter-graph-a9ea4-7dbf9cdd7d from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: splitter-graph-a9ea4.18c68133b390c84b namespace: kserve-ci-e2e-test resourceVersion: "26099" uid: 9bc2eb6c-885e-4360-9fa4-b7f9175eff48 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: splitter-graph-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "26142" uid: 52655fd1-8fb6-4003-a898-30b8bc2e0e4b kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: InferenceGraph [splitter-graph-a9ea4] is Ready metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:41Z" name: splitter-graph-a9ea4.18c68133b912dc68 namespace: kserve-ci-e2e-test resourceVersion: "26144" uid: 6ee8f539-4971-4858-802e-8db6cc1335e9 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:11:41.240996Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15405" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-021cc-predictor-7b688f7777-fsnkl to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e4eec8335 namespace: kserve-ci-e2e-test resourceVersion: "15410" uid: 1e9667b2-3491-490b-aade-129f5bbfa75d reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15417" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Add eth0 [10.134.0.33/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e6940263d namespace: kserve-ci-e2e-test resourceVersion: "15467" uid: 29783653-0d9c-4796-a772-737257a8396b reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e6aa7f90e namespace: kserve-ci-e2e-test resourceVersion: "15469" uid: be819bce-798e-421e-bac3-09d5e8790dfe reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e6ee9e599 namespace: kserve-ci-e2e-test resourceVersion: "15472" uid: 119dfc41-871d-4b9e-8b47-d698ed2d7822 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e6f5b6a1a namespace: kserve-ci-e2e-test resourceVersion: "15474" uid: 2c99167a-fd77-483b-95e7-cc23e4408b90 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e6f681423 namespace: kserve-ci-e2e-test resourceVersion: "15475" uid: 64c791b0-1af8-4ec1-ad09-dbcb5ec40c2b reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e730d5ba8 namespace: kserve-ci-e2e-test resourceVersion: "15476" uid: 94d9c494-4f84-4f57-9ea7-5892acbf41a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e738353b5 namespace: kserve-ci-e2e-test resourceVersion: "15477" uid: 4cc5c2da-5035-4523-a4c7-fbf226b79f8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:20:58Z" message: 'Readiness probe failed: dial tcp 10.134.0.33:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:58Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6800e94e1fbfd namespace: kserve-ci-e2e-test resourceVersion: "19898" uid: b690a028-f6ff-4291-871c-823d640cc016 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6808f82766de4 namespace: kserve-ci-e2e-test resourceVersion: "19804" uid: 91cf5aa7-118d-4441-ab69-c6dfe64a604f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c6808f8276e881 namespace: kserve-ci-e2e-test resourceVersion: "19809" uid: f6d4709d-1bd6-4b2b-a9f6-6ebc85962756 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl namespace: kserve-ci-e2e-test resourceVersion: "15408" uid: 5deadc31-746f-49fe-a654-d1f27457af11 kind: Event lastTimestamp: "2026-07-28T16:20:58Z" message: 'Readiness probe failed: Get "https://10.134.0.33:8643/healthz": dial tcp 10.134.0.33:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:58Z" name: success-200-isvc-021cc-predictor-7b688f7777-fsnkl.18c68090097aafbd namespace: kserve-ci-e2e-test resourceVersion: "19896" uid: 2062a4d9-a613-49a8-8ab0-f9d8f6cc3928 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-021cc-predictor-7b688f7777 namespace: kserve-ci-e2e-test resourceVersion: "15387" uid: b57a3209-7366-404c-85a9-6f271ae897a2 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: 'Created pod: success-200-isvc-021cc-predictor-7b688f7777-fsnkl' metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor-7b688f7777.18c6800e4e4f9a04 namespace: kserve-ci-e2e-test resourceVersion: "15407" uid: fc4f284c-69e7-40d2-8444-53e8831e5217 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15380" uid: 8663aff7-08b7-43c6-b072-0276826b6cef kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Scaled up replica set success-200-isvc-021cc-predictor-7b688f7777 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-021cc-predictor.18c6800e4ac6f0d1 namespace: kserve-ci-e2e-test resourceVersion: "15391" uid: 1b47e568-8ace-4296-8923-4334d56b0ec9 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: ad633c30-40f8-4dc8-9650-fe5101cc8ccc kind: Event lastTimestamp: "2026-07-28T16:12:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:11Z" name: success-200-isvc-021cc-predictor.18c68011c94ef19d namespace: kserve-ci-e2e-test resourceVersion: "15768" uid: 09161a91-23cd-4e20-9fd6-62335b2b34ef reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:11:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15385" uid: ad633c30-40f8-4dc8-9650-fe5101cc8ccc kind: Event lastTimestamp: "2026-07-28T16:12:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:11:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:11Z" name: success-200-isvc-021cc-predictor.18c68011c9cb62e6 namespace: kserve-ci-e2e-test resourceVersion: "15769" uid: 3a0fd789-396f-4395-aa64-f6c52c67b3ac reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: ad633c30-40f8-4dc8-9650-fe5101cc8ccc kind: Event lastTimestamp: "2026-07-28T16:12:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:56Z" name: success-200-isvc-021cc-predictor.18c68018c7515425 namespace: kserve-ci-e2e-test resourceVersion: "16357" uid: 46c86f27-cf4d-4c7c-9a94-860f20148bdb reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:12:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-021cc-predictor namespace: kserve-ci-e2e-test resourceVersion: "15623" uid: ad633c30-40f8-4dc8-9650-fe5101cc8ccc kind: Event lastTimestamp: "2026-07-28T16:12:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:12:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:56Z" name: success-200-isvc-021cc-predictor.18c68018c7c0428c namespace: kserve-ci-e2e-test resourceVersion: "16358" uid: 3cd13edb-5a10-42d3-aad1-47c4498071ce reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15373" uid: 8ff1353c-21f3-4afd-aa4d-e97c8e5ecaf3 kind: Event lastTimestamp: "2026-07-28T16:12:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:28Z" name: success-200-isvc-021cc.18c6800e4ae9c3cc namespace: kserve-ci-e2e-test resourceVersion: "15915" uid: 145b18ca-3212-44df-8a99-33eeef42a4e3 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-021cc namespace: kserve-ci-e2e-test resourceVersion: "15913" uid: 8ff1353c-21f3-4afd-aa4d-e97c8e5ecaf3 kind: Event lastTimestamp: "2026-07-28T16:12:28Z" message: InferenceService [success-200-isvc-021cc] is Ready metadata: creationTimestamp: "2026-07-28T16:12:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:28Z" name: success-200-isvc-021cc.18c6801951647c34 namespace: kserve-ci-e2e-test resourceVersion: "15914" uid: 411241bc-618a-48dc-b226-979b0e7e6463 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:12:51.256814Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16226" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801e9c314504 namespace: kserve-ci-e2e-test resourceVersion: "16234" uid: 99bae9a3-4430-4218-9b96-d55d503c5f20 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16244" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Add eth0 [10.134.0.37/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801eb69eec49 namespace: kserve-ci-e2e-test resourceVersion: "16295" uid: bfb21989-96c3-4f7c-ae55-1ccbbccef833 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801eb8f0accd namespace: kserve-ci-e2e-test resourceVersion: "16299" uid: 6948ff58-488e-4c56-b520-10fc9ea7b39f reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ebca65227 namespace: kserve-ci-e2e-test resourceVersion: "16301" uid: 48591076-2133-4c7a-aaa3-76154f801ae9 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ebd2a07f8 namespace: kserve-ci-e2e-test resourceVersion: "16302" uid: 6a7bf349-0021-46dd-b55b-76e9c1dcdb59 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ebd3627b4 namespace: kserve-ci-e2e-test resourceVersion: "16303" uid: 99ace4c0-d26a-4344-90a8-ce7adedfd7a2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ec0b65f14 namespace: kserve-ci-e2e-test resourceVersion: "16305" uid: 164dcd45-b673-4ab2-b8b0-f5567e0917a4 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ec1388cbd namespace: kserve-ci-e2e-test resourceVersion: "16307" uid: c8b4c2dd-7ac8-485b-acd9-7017424996c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:12:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:22:08Z" message: 'Readiness probe failed: dial tcp 10.134.0.37:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:12:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:08Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6801ef6ed6b93 namespace: kserve-ci-e2e-test resourceVersion: "20624" uid: 513b8351-75fa-4770-b2b9-d1ffa53f3091 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6809fc875c085 namespace: kserve-ci-e2e-test resourceVersion: "20468" uid: 8172a95f-4c27-45d4-b2cb-846d3d1ec812 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c6809fc87c7172 namespace: kserve-ci-e2e-test resourceVersion: "20470" uid: 01ad905e-6e4e-43af-ac7b-71f5ac2c1f2c reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q namespace: kserve-ci-e2e-test resourceVersion: "16230" uid: e1de457f-a67f-40b2-98de-c159c23a9a0a kind: Event lastTimestamp: "2026-07-28T16:22:08Z" message: 'Readiness probe failed: Get "https://10.134.0.37:8643/healthz": dial tcp 10.134.0.37:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:08Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q.18c680a06b6096ec namespace: kserve-ci-e2e-test resourceVersion: "20621" uid: d7d1ebf8-aab7-4c10-a26e-51f8df370653 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-0402e-predictor-5f7c85c4d9 namespace: kserve-ci-e2e-test resourceVersion: "16208" uid: 5b37fa4e-e873-4f53-82cf-eae2aa2ff0d3 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: 'Created pod: success-200-isvc-0402e-predictor-5f7c85c4d9-m8d6q' metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor-5f7c85c4d9.18c6801e9b7633ea namespace: kserve-ci-e2e-test resourceVersion: "16228" uid: a28f6606-7f72-464a-9762-40726b560ae5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16205" uid: d9922800-ee97-47f9-89cc-b643eb924d83 kind: Event lastTimestamp: "2026-07-28T16:12:51Z" message: Scaled up replica set success-200-isvc-0402e-predictor-5f7c85c4d9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:51Z" name: success-200-isvc-0402e-predictor.18c6801e99aadb57 namespace: kserve-ci-e2e-test resourceVersion: "16214" uid: d3ad1330-3b66-4fb1-bd25-3b0506d27d10 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 48199d06-5aa2-4025-839d-1119254b7aab kind: Event lastTimestamp: "2026-07-28T16:13:06Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:06Z" name: success-200-isvc-0402e-predictor.18c68022187fcc7a namespace: kserve-ci-e2e-test resourceVersion: "16447" uid: d563352d-ac2a-4554-b763-1fa27deff3ad reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16207" uid: 48199d06-5aa2-4025-839d-1119254b7aab kind: Event lastTimestamp: "2026-07-28T16:13:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:13:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:06Z" name: success-200-isvc-0402e-predictor.18c6802218fa31c5 namespace: kserve-ci-e2e-test resourceVersion: "16449" uid: 8e2b0bab-f007-46ac-9628-cbabd59bfa60 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 48199d06-5aa2-4025-839d-1119254b7aab kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:51Z" name: success-200-isvc-0402e-predictor.18c6802597d5968a namespace: kserve-ci-e2e-test resourceVersion: "16818" uid: cbb92a12-08cb-4ba6-a7ac-c86016055201 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:13:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-0402e-predictor namespace: kserve-ci-e2e-test resourceVersion: "16448" uid: 48199d06-5aa2-4025-839d-1119254b7aab kind: Event lastTimestamp: "2026-07-28T16:13:51Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:51Z" name: success-200-isvc-0402e-predictor.18c68025984ee691 namespace: kserve-ci-e2e-test resourceVersion: "16821" uid: f0884d67-c178-4924-a40d-348a1188467f reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:12:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0402e namespace: kserve-ci-e2e-test resourceVersion: "16183" uid: c0c04b35-6b1f-4c35-9df1-7bbc4052390a kind: Event lastTimestamp: "2026-07-28T16:13:39Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:12:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:13:39Z" name: success-200-isvc-0402e.18c6801e99b30c9d namespace: kserve-ci-e2e-test resourceVersion: "16719" uid: 64dc6c4d-b3c3-43e1-b128-788d873836eb reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:38Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-0402e namespace: kserve-ci-e2e-test resourceVersion: "16715" uid: c0c04b35-6b1f-4c35-9df1-7bbc4052390a kind: Event lastTimestamp: "2026-07-28T16:13:38Z" message: InferenceService [success-200-isvc-0402e] is Ready metadata: creationTimestamp: "2026-07-28T16:13:38Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:13:38Z" name: success-200-isvc-0402e.18c68029b7127958 namespace: kserve-ci-e2e-test resourceVersion: "16716" uid: 218623ff-95d7-4a4f-acf2-5ade01701b26 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:10:06.864082Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14298" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-3692d-predictor-5b97fbf46c-6tfht to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:10:06Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ff8559f382f namespace: kserve-ci-e2e-test resourceVersion: "14306" uid: bc41148f-d348-417e-8e5e-5aab5b0a64a2 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14311" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Add eth0 [10.134.0.30/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:10:07Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ff87059442e namespace: kserve-ci-e2e-test resourceVersion: "14372" uid: 84162ba7-eaf0-429c-9da9-8d980ceff799 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:07Z" message: Pulling image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" metadata: creationTimestamp: "2026-07-28T16:10:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:07Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ff8724bf8f8 namespace: kserve-ci-e2e-test resourceVersion: "14382" uid: 02bf909c-bf96-41f3-b908-d427550f40c7 reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: 'Successfully pulled image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" in 16.38s (16.38s including waiting). Image size: 1334995039 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffc42a58047 namespace: kserve-ci-e2e-test resourceVersion: "14627" uid: 8539fa85-c0cf-4381-81d1-c54fa6af3671 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffc46ee92fe namespace: kserve-ci-e2e-test resourceVersion: "14629" uid: 2a26ebd2-9dc8-4dce-9b7c-5b0f3f231b28 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffc47707245 namespace: kserve-ci-e2e-test resourceVersion: "14631" uid: 216ac412-5d0d-4554-9f0c-5d52cc9e2ba2 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:23Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:23Z" message: Pulling image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" metadata: creationTimestamp: "2026-07-28T16:10:23Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:23Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffc477ea7ac namespace: kserve-ci-e2e-test resourceVersion: "14633" uid: b26f09e4-bb33-4ea0-94f5-2ec56ff6c1eb reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: 'Successfully pulled image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" in 2.703s (2.703s including waiting). Image size: 211946088 bytes.' metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffce8a31733 namespace: kserve-ci-e2e-test resourceVersion: "14657" uid: 78f4bd89-d1cf-4e5a-b048-ba3209bdbea1 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffcecc684fd namespace: kserve-ci-e2e-test resourceVersion: "14659" uid: 5396a267-15fe-4d64-a344-1cb8223a6035 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:10:26Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:10:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:10:26Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffced4201c9 namespace: kserve-ci-e2e-test resourceVersion: "14661" uid: 6da2b57b-729e-49c4-bb6e-b3f8c7c1f449 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:10:28Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:11:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.30:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:10:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:44Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c67ffd443ed4aa namespace: kserve-ci-e2e-test resourceVersion: "15512" uid: 18805ce8-06be-40e0-a4d5-79832fb6e8ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c6800e4c292284 namespace: kserve-ci-e2e-test resourceVersion: "15399" uid: 7fa678d1-613a-4ef3-b9c8-59f4249ac4ef reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c6800e4c29a449 namespace: kserve-ci-e2e-test resourceVersion: "15400" uid: 4817647f-8f55-4d60-87df-e3f53be343ec reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht namespace: kserve-ci-e2e-test resourceVersion: "14301" uid: bc7d8d41-d9ba-4512-9540-fd814b045f09 kind: Event lastTimestamp: "2026-07-28T16:11:44Z" message: 'Readiness probe failed: Get "https://10.134.0.30:8643/healthz": dial tcp 10.134.0.30:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:11:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:44Z" name: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht.18c6800ef673645a namespace: kserve-ci-e2e-test resourceVersion: "15511" uid: a5f88ae5-5731-4642-8fc0-4eaca71f4832 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-3692d-predictor-5b97fbf46c namespace: kserve-ci-e2e-test resourceVersion: "14287" uid: ced26657-c68f-438c-a333-b43089d41fa5 kind: Event lastTimestamp: "2026-07-28T16:10:06Z" message: 'Created pod: success-200-isvc-3692d-predictor-5b97fbf46c-6tfht' metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:06Z" name: success-200-isvc-3692d-predictor-5b97fbf46c.18c67ff8552b6e9a namespace: kserve-ci-e2e-test resourceVersion: "14300" uid: 02811648-be8a-4993-b5ea-81a5ae0abd8d reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14286" uid: abda4b6c-96ac-4f30-acd2-48337def032a kind: Event lastTimestamp: "2026-07-28T16:10:06Z" message: Scaled up replica set success-200-isvc-3692d-predictor-5b97fbf46c from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:06Z" name: success-200-isvc-3692d-predictor.18c67ff84e714314 namespace: kserve-ci-e2e-test resourceVersion: "14289" uid: b404fe7f-61f2-49e0-a43a-44cf432426f8 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:10:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 70fcb5e6-8bdd-42bf-a55e-45dce52e9d6e kind: Event lastTimestamp: "2026-07-28T16:10:36Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:36Z" name: success-200-isvc-3692d-predictor.18c67ffbd3f0b3fd namespace: kserve-ci-e2e-test resourceVersion: "14742" uid: b86295a4-e159-44ce-ab13-1bb1e78f8937 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:10:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14295" uid: 70fcb5e6-8bdd-42bf-a55e-45dce52e9d6e kind: Event lastTimestamp: "2026-07-28T16:10:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:10:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:10:36Z" name: success-200-isvc-3692d-predictor.18c67ffbd451d9f6 namespace: kserve-ci-e2e-test resourceVersion: "14743" uid: 36fcace1-80a6-494b-a66a-71381fac571b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 70fcb5e6-8bdd-42bf-a55e-45dce52e9d6e kind: Event lastTimestamp: "2026-07-28T16:11:36Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:36Z" name: success-200-isvc-3692d-predictor.18c68002d320e6dd namespace: kserve-ci-e2e-test resourceVersion: "15307" uid: 08e0b7cf-138b-44d5-a280-984d2feb7515 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:10:51Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-3692d-predictor namespace: kserve-ci-e2e-test resourceVersion: "14599" uid: 70fcb5e6-8bdd-42bf-a55e-45dce52e9d6e kind: Event lastTimestamp: "2026-07-28T16:11:36Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:10:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:36Z" name: success-200-isvc-3692d-predictor.18c68002d3796fd6 namespace: kserve-ci-e2e-test resourceVersion: "15308" uid: 7a0fac49-241a-4e6d-abe5-ad9fa76889aa reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:10:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: a46ecfd5-948d-4e98-9b66-4289cb979533 kind: Event lastTimestamp: "2026-07-28T16:11:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:14Z" name: success-200-isvc-3692d.18c67ff854944ea0 namespace: kserve-ci-e2e-test resourceVersion: "15093" uid: 7c0fc24d-3184-4eb7-ae4f-af065200ee76 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: a46ecfd5-948d-4e98-9b66-4289cb979533 kind: Event lastTimestamp: "2026-07-28T16:10:06Z" message: 'Failed to update status for InferenceService "success-200-isvc-3692d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3692d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:10:06Z" name: success-200-isvc-3692d.18c67ff85533296d namespace: kserve-ci-e2e-test resourceVersion: "14302" uid: 721a0d05-9876-491d-977a-bfd0b2757176 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:10:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "14279" uid: a46ecfd5-948d-4e98-9b66-4289cb979533 kind: Event lastTimestamp: "2026-07-28T16:10:06Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-3692d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:10:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:10:06Z" name: success-200-isvc-3692d.18c67ff855337d4d namespace: kserve-ci-e2e-test resourceVersion: "14305" uid: 422749ae-8477-481d-be13-2fe45c38177c reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-3692d namespace: kserve-ci-e2e-test resourceVersion: "15090" uid: a46ecfd5-948d-4e98-9b66-4289cb979533 kind: Event lastTimestamp: "2026-07-28T16:11:14Z" message: InferenceService [success-200-isvc-3692d] is Ready metadata: creationTimestamp: "2026-07-28T16:11:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:14Z" name: success-200-isvc-3692d.18c6800800109bf9 namespace: kserve-ci-e2e-test resourceVersion: "15091" uid: a4fe582d-d875-48ac-b62a-4d4169db3205 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:51.619767Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26369" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c6813617664638 namespace: kserve-ci-e2e-test resourceVersion: "26374" uid: d9dddf18-37e2-46a2-bc22-5a15e9c038e0 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26380" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Add eth0 [10.134.0.51/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c6813631fba377 namespace: kserve-ci-e2e-test resourceVersion: "26432" uid: 2817b0b0-9c5f-419f-9269-af9f802d08fe reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681363371e54d namespace: kserve-ci-e2e-test resourceVersion: "26434" uid: 207176f7-2204-4b74-ba6d-2810cec63fde reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c6813637b210a5 namespace: kserve-ci-e2e-test resourceVersion: "26435" uid: 2e80d288-ad1a-4002-8e26-f8967b3db78a reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c68136383640f1 namespace: kserve-ci-e2e-test resourceVersion: "26436" uid: 80f6e98d-29d8-43fe-8e93-bb6539aa6739 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c6813638426d2f namespace: kserve-ci-e2e-test resourceVersion: "26437" uid: 20ca9d9d-78ab-4fa6-92f9-eda202c5e25d reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681363be2248f namespace: kserve-ci-e2e-test resourceVersion: "26441" uid: 6aaef7ec-eed5-411f-835a-a65eeec60e92 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:32:52Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:52Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681363c622448 namespace: kserve-ci-e2e-test resourceVersion: "26443" uid: 0bbebdab-7dcf-4fd0-a2a3-cdc68a4d249b reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T16:32:52Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:42:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.51:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:52Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:18Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681365dd64404 namespace: kserve-ci-e2e-test resourceVersion: "30613" uid: f3f8951e-563e-4ef0-b304-2f0913572344 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:42:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:16Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681b99e0e2365 namespace: kserve-ci-e2e-test resourceVersion: "30580" uid: 143b0eec-bc0a-43dc-83ef-69a87683a6d3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:42:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:42:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:16Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681b99e0ea123 namespace: kserve-ci-e2e-test resourceVersion: "30581" uid: 23706f1c-9804-4845-8579-f49fdb6e5e88 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:42:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg namespace: kserve-ci-e2e-test resourceVersion: "26371" uid: addebf89-052d-4d13-9c5c-0fef273bb273 kind: Event lastTimestamp: "2026-07-28T16:42:18Z" message: 'Readiness probe failed: Get "https://10.134.0.51:8643/healthz": dial tcp 10.134.0.51:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:42:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:42:18Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg.18c681ba266dd5d1 namespace: kserve-ci-e2e-test resourceVersion: "30612" uid: 60b957c8-4f82-4a94-b124-9b81fd5e2791 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-49c25-predictor-75f8fbbd8f namespace: kserve-ci-e2e-test resourceVersion: "26357" uid: 3cd385c8-059e-4002-a8eb-3482555db0a7 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'Created pod: success-200-isvc-49c25-predictor-75f8fbbd8f-bsnmg' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-49c25-predictor-75f8fbbd8f.18c6813616eab04e namespace: kserve-ci-e2e-test resourceVersion: "26370" uid: 2485d587-4daf-43ca-bae3-a45ab671ca51 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26351" uid: af442161-a34a-4480-b9b4-c9379d1820d4 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Scaled up replica set success-200-isvc-49c25-predictor-75f8fbbd8f from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-49c25-predictor.18c68136151e09e0 namespace: kserve-ci-e2e-test resourceVersion: "26360" uid: e0f7ed0b-3b28-482c-8175-7bc8ccfec549 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 78f8a8f5-95d7-403f-9990-61bf1ee30f06 kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: success-200-isvc-49c25-predictor.18c68139941d2871 namespace: kserve-ci-e2e-test resourceVersion: "26718" uid: 9dfbf23d-c481-4951-bb02-448bae967e64 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:33:06Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26356" uid: 78f8a8f5-95d7-403f-9990-61bf1ee30f06 kind: Event lastTimestamp: "2026-07-28T16:33:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:21Z" name: success-200-isvc-49c25-predictor.18c6813994895d7c namespace: kserve-ci-e2e-test resourceVersion: "26719" uid: 23c47218-0c84-453a-a3f6-e08d3730e367 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 78f8a8f5-95d7-403f-9990-61bf1ee30f06 kind: Event lastTimestamp: "2026-07-28T16:34:06Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:06Z" name: success-200-isvc-49c25-predictor.18c6814092859704 namespace: kserve-ci-e2e-test resourceVersion: "27174" uid: c07c67ea-3ab7-496e-84c2-83f40f5fb6b6 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-49c25-predictor namespace: kserve-ci-e2e-test resourceVersion: "26588" uid: 78f8a8f5-95d7-403f-9990-61bf1ee30f06 kind: Event lastTimestamp: "2026-07-28T16:34:06Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:06Z" name: success-200-isvc-49c25-predictor.18c6814092eb75b0 namespace: kserve-ci-e2e-test resourceVersion: "27175" uid: 27097c7d-37cc-4c49-b962-b31ac34d3100 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 94526626-4041-41e0-9305-6fb086ca8188 kind: Event lastTimestamp: "2026-07-28T16:33:48Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:48Z" name: success-200-isvc-49c25.18c6813615556914 namespace: kserve-ci-e2e-test resourceVersion: "27002" uid: 2387e03b-6691-46cf-8b3d-cbce935d953b reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 94526626-4041-41e0-9305-6fb086ca8188 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'Failed to update status for InferenceService "success-200-isvc-49c25": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-49c25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-49c25.18c6813615e16be3 namespace: kserve-ci-e2e-test resourceVersion: "26365" uid: 63d3e381-4971-4db1-84d5-77554f5a417a reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "26337" uid: 94526626-4041-41e0-9305-6fb086ca8188 kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-49c25": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-49c25.18c6813615e1d41e namespace: kserve-ci-e2e-test resourceVersion: "26368" uid: e2ab2949-705e-4ebc-8a8a-2df844fddc1d reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:48Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-49c25 namespace: kserve-ci-e2e-test resourceVersion: "27000" uid: 94526626-4041-41e0-9305-6fb086ca8188 kind: Event lastTimestamp: "2026-07-28T16:33:48Z" message: InferenceService [success-200-isvc-49c25] is Ready metadata: creationTimestamp: "2026-07-28T16:33:48Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:48Z" name: success-200-isvc-49c25.18c681436deb9440 namespace: kserve-ci-e2e-test resourceVersion: "27001" uid: fc5f4e6b-a5e8-4208-a29b-de949520a41c reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:22:06.128888Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20489" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-7c1b1-predictor-5f679bf9f-px27h to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c6809fcd250ccb namespace: kserve-ci-e2e-test resourceVersion: "20497" uid: 03ccd4e9-d2f0-4c28-b44a-b360f0df4b06 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-7c1b1-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c6809fd622dfac namespace: kserve-ci-e2e-test resourceVersion: "20552" uid: 055cbd61-5161-48d3-aca9-e0f0a6b46e05 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20511" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Add eth0 [10.134.0.41/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a00b547e77 namespace: kserve-ci-e2e-test resourceVersion: "20587" uid: a9116808-b408-42b4-8a5e-4dae920a1150 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a00cdd56bb namespace: kserve-ci-e2e-test resourceVersion: "20589" uid: 3f83a329-90d1-4f50-b6ce-9d4c64d76a05 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a010389b15 namespace: kserve-ci-e2e-test resourceVersion: "20594" uid: fc7bc9e1-fafd-4daa-bca1-af1524ca216f reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a010b5ff00 namespace: kserve-ci-e2e-test resourceVersion: "20595" uid: 165e6526-e041-4f74-ba8b-eb0002f9c4a7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a010c0e60d namespace: kserve-ci-e2e-test resourceVersion: "20596" uid: 1aa99f5b-2eb9-4db6-a877-451b3ed4afd8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a01489d4be namespace: kserve-ci-e2e-test resourceVersion: "20600" uid: af1d90e7-9904-4f28-9e7d-8feec02fa7ad reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:07Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:07Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:07Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:07Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a0151a9f73 namespace: kserve-ci-e2e-test resourceVersion: "20601" uid: d1ea6a79-fef7-4df5-be04-806134902e8d reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:22:08Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:22:54Z" message: 'Readiness probe failed: dial tcp 10.134.0.41:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:08Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:54Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680a05a3d4026 namespace: kserve-ci-e2e-test resourceVersion: "21164" uid: 4a7b82be-2d3a-49d2-b223-7a5543ee9717 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680b27c140c78 namespace: kserve-ci-e2e-test resourceVersion: "21536" uid: 00500328-fc4a-4799-9d30-7fcd8f009d64 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680b27c154acc namespace: kserve-ci-e2e-test resourceVersion: "21539" uid: 6e329ec1-c086-4d8a-a55b-8054398480f3 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:29Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h namespace: kserve-ci-e2e-test resourceVersion: "20492" uid: f8e0ce44-1af6-49d1-80f6-2edca34b926a kind: Event lastTimestamp: "2026-07-28T16:23:29Z" message: 'Readiness probe failed: Get "https://10.134.0.41:8643/healthz": dial tcp 10.134.0.41:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:23:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:29Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h.18c680b33698e5ea namespace: kserve-ci-e2e-test resourceVersion: "21677" uid: 95db78e0-9030-4cec-ae24-d68f248271a7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-7c1b1-predictor-5f679bf9f namespace: kserve-ci-e2e-test resourceVersion: "20469" uid: 5924630d-99b4-4420-bdcf-9da60f2a5ea3 kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: 'Created pod: success-200-isvc-7c1b1-predictor-5f679bf9f-px27h' metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-7c1b1-predictor-5f679bf9f.18c6809fcc9d8c1d namespace: kserve-ci-e2e-test resourceVersion: "20491" uid: 5a0eec6c-0dc0-4ed3-a72d-d048a538de0a reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20467" uid: 8c6b93d9-a637-422d-bd31-c68b52c685eb kind: Event lastTimestamp: "2026-07-28T16:22:06Z" message: Scaled up replica set success-200-isvc-7c1b1-predictor-5f679bf9f from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:06Z" name: success-200-isvc-7c1b1-predictor.18c6809fc9b29ae3 namespace: kserve-ci-e2e-test resourceVersion: "20473" uid: 3cca148c-012e-422c-bfdb-c55feb327ce2 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: fd433e56-20e6-46e9-a76b-9a70d6caeeb5 kind: Event lastTimestamp: "2026-07-28T16:22:21Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:21Z" name: success-200-isvc-7c1b1-predictor.18c680a3491a7bd3 namespace: kserve-ci-e2e-test resourceVersion: "20878" uid: b7f9eb2e-c910-4baa-b0b4-7fa91b4c3c69 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:21Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20471" uid: fd433e56-20e6-46e9-a76b-9a70d6caeeb5 kind: Event lastTimestamp: "2026-07-28T16:22:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:21Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:21Z" name: success-200-isvc-7c1b1-predictor.18c680a34980f3bb namespace: kserve-ci-e2e-test resourceVersion: "20880" uid: 255c510e-d279-4166-bcb2-7254ed04240d reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: fd433e56-20e6-46e9-a76b-9a70d6caeeb5 kind: Event lastTimestamp: "2026-07-28T16:23:21Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:21Z" name: success-200-isvc-7c1b1-predictor.18c680a6c88d3753 namespace: kserve-ci-e2e-test resourceVersion: "21436" uid: a921fedc-2eec-4aea-9511-1272cd4b1c88 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 4 eventTime: null firstTimestamp: "2026-07-28T16:22:36Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-7c1b1-predictor namespace: kserve-ci-e2e-test resourceVersion: "20879" uid: fd433e56-20e6-46e9-a76b-9a70d6caeeb5 kind: Event lastTimestamp: "2026-07-28T16:23:21Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:22:36Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:21Z" name: success-200-isvc-7c1b1-predictor.18c680a6c8e5f697 namespace: kserve-ci-e2e-test resourceVersion: "21437" uid: c9ca3526-7da8-44c4-9353-eec9afa15b4c reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:06Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "20447" uid: 4096921f-62d6-4139-94ea-b951fb11b05a kind: Event lastTimestamp: "2026-07-28T16:23:04Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:22:06Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:04Z" name: success-200-isvc-7c1b1.18c6809fcb614bc2 namespace: kserve-ci-e2e-test resourceVersion: "21259" uid: c5c84060-c094-4e7b-9ee9-93e5778a19ef reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:04Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-7c1b1 namespace: kserve-ci-e2e-test resourceVersion: "21257" uid: 4096921f-62d6-4139-94ea-b951fb11b05a kind: Event lastTimestamp: "2026-07-28T16:23:04Z" message: InferenceService [success-200-isvc-7c1b1] is Ready metadata: creationTimestamp: "2026-07-28T16:23:04Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:04Z" name: success-200-isvc-7c1b1.18c680ad6a236d8e namespace: kserve-ci-e2e-test resourceVersion: "21258" uid: bf6bf3d4-a6ee-4ebb-80b5-d28c6622618e reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:31:41.380720Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25186" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125bcd3760d namespace: kserve-ci-e2e-test resourceVersion: "25196" uid: b931549e-c040-4462-a76e-8b13e9287b69 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25206" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Add eth0 [10.134.0.47/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125d7e37bc7 namespace: kserve-ci-e2e-test resourceVersion: "25257" uid: 06d77d9c-926c-48af-8181-d5302cff6522 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125d9627dda namespace: kserve-ci-e2e-test resourceVersion: "25259" uid: 37dbb426-60e0-4b4e-94a4-296ec188d8e2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125dcde3279 namespace: kserve-ci-e2e-test resourceVersion: "25260" uid: fb87e01f-33cd-49b5-b012-9df7347f09cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125dd6551ec namespace: kserve-ci-e2e-test resourceVersion: "25261" uid: 1d598d2f-9fa2-4919-84ea-01d90c0dd660 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125dd71847c namespace: kserve-ci-e2e-test resourceVersion: "25263" uid: 7c1acf7a-d2f8-4880-bcc7-b1b19aab086a reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125e0bb51af namespace: kserve-ci-e2e-test resourceVersion: "25267" uid: 7237e390-3b6e-4bea-b0d1-43b01c9265cc reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68125e13b591a namespace: kserve-ci-e2e-test resourceVersion: "25268" uid: 221f4f75-4277-4330-bcfd-a72d0fb2cd96 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:31:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:32:19Z" message: 'Readiness probe failed: dial tcp 10.134.0.47:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:31:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:19Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c681263680b46d namespace: kserve-ci-e2e-test resourceVersion: "25856" uid: a5e81384-b54c-4d51-80e0-b8a580cb90fc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c6813615859c89 namespace: kserve-ci-e2e-test resourceVersion: "26363" uid: 526c2b93-ad41-4a67-acf3-5072fd97c01d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:51Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:32:51Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:51Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:51Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c6813615861ead namespace: kserve-ci-e2e-test resourceVersion: "26364" uid: d67e8ed5-c70b-4e67-a4eb-3386ece8f127 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:54Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc namespace: kserve-ci-e2e-test resourceVersion: "25188" uid: 7abc91bb-7c02-4a45-9f77-54df554fe14e kind: Event lastTimestamp: "2026-07-28T16:32:54Z" message: 'Readiness probe failed: Get "https://10.134.0.47:8643/healthz": dial tcp 10.134.0.47:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:54Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:54Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc.18c68136bea95906 namespace: kserve-ci-e2e-test resourceVersion: "26477" uid: ad42f253-fc66-4c39-8426-97fa9086fb7f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-a9ea4-predictor-64d69d9c55 namespace: kserve-ci-e2e-test resourceVersion: "25171" uid: 46186735-f303-4e19-a682-a8f9d9eda0c9 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Created pod: success-200-isvc-a9ea4-predictor-64d69d9c55-2bfwc' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor-64d69d9c55.18c68125bc1db783 namespace: kserve-ci-e2e-test resourceVersion: "25189" uid: dd5e190c-cd5a-4d3f-bbe3-acbb07ef67a0 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25169" uid: b5070c53-ba29-4a20-9f89-6187ec421109 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Scaled up replica set success-200-isvc-a9ea4-predictor-64d69d9c55 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4-predictor.18c68125b784787c namespace: kserve-ci-e2e-test resourceVersion: "25173" uid: 0ea60103-a0a0-46ff-bfc9-74200f5ac625 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25170" uid: 697a3e93-3670-4085-8fcf-733c4ebbc3b5 kind: Event lastTimestamp: "2026-07-28T16:32:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:11Z" name: success-200-isvc-a9ea4-predictor.18c6812935e48e63 namespace: kserve-ci-e2e-test resourceVersion: "25595" uid: 30657dfa-0ecc-41e1-81c6-406fa59ef248 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:31:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25170" uid: 697a3e93-3670-4085-8fcf-733c4ebbc3b5 kind: Event lastTimestamp: "2026-07-28T16:32:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:31:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:11Z" name: success-200-isvc-a9ea4-predictor.18c6812936463673 namespace: kserve-ci-e2e-test resourceVersion: "25596" uid: e271b12c-1c69-4054-b84a-400a75a2154e reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:32:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: 697a3e93-3670-4085-8fcf-733c4ebbc3b5 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-a9ea4-predictor.18c6813034773dc1 namespace: kserve-ci-e2e-test resourceVersion: "26109" uid: 86cfcd4b-667a-4902-a086-0595dc6e8304 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:32:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-a9ea4-predictor namespace: kserve-ci-e2e-test resourceVersion: "25399" uid: 697a3e93-3670-4085-8fcf-733c4ebbc3b5 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:32:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-a9ea4-predictor.18c6813034e3dc58 namespace: kserve-ci-e2e-test resourceVersion: "26116" uid: 85c0f5a9-a9b0-4d1f-8e3d-9e3f2b9673bb reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 64e71169-6b6f-4b15-adaa-bcc9df22b800 kind: Event lastTimestamp: "2026-07-28T16:32:29Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:29Z" name: success-200-isvc-a9ea4.18c68125b7871dcd namespace: kserve-ci-e2e-test resourceVersion: "25961" uid: 4214ff16-e0ea-4b03-a1cd-8fb03de0bdb7 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 64e71169-6b6f-4b15-adaa-bcc9df22b800 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-a9ea4": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9ea4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4.18c68125b842d44b namespace: kserve-ci-e2e-test resourceVersion: "25175" uid: 2f7952ee-c8cf-4a32-a3ad-acf6268777ac reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25160" uid: 64e71169-6b6f-4b15-adaa-bcc9df22b800 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-a9ea4": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-a9ea4.18c68125b8433159 namespace: kserve-ci-e2e-test resourceVersion: "25176" uid: 014f0097-ced9-446e-b29f-240e95f5ac94 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:29Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-a9ea4 namespace: kserve-ci-e2e-test resourceVersion: "25957" uid: 64e71169-6b6f-4b15-adaa-bcc9df22b800 kind: Event lastTimestamp: "2026-07-28T16:32:29Z" message: InferenceService [success-200-isvc-a9ea4] is Ready metadata: creationTimestamp: "2026-07-28T16:32:29Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:29Z" name: success-200-isvc-a9ea4.18c68130f5a073fb namespace: kserve-ci-e2e-test resourceVersion: "25958" uid: b8a4c74c-1d97-4078-930d-1f86246b6c6a reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:23:26.454066Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21549" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-c3dc2-predictor-58b487689c-z5qjm to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b280e601cc namespace: kserve-ci-e2e-test resourceVersion: "21561" uid: e2165a1a-e3ac-40ed-a89d-c4a287827519 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21569" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Add eth0 [10.134.0.45/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b29b87900b namespace: kserve-ci-e2e-test resourceVersion: "21623" uid: 6f8455f4-7b47-40fc-983f-5691694b2cc5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b29d0d370b namespace: kserve-ci-e2e-test resourceVersion: "21627" uid: e6d75030-c448-4001-ae0a-42d4a027b1c8 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2a1cefaba namespace: kserve-ci-e2e-test resourceVersion: "21633" uid: eb2f1cec-40c8-4d9f-8722-c7585172fc6e reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2a25f484c namespace: kserve-ci-e2e-test resourceVersion: "21634" uid: 470a10ea-7044-4040-9518-6bd4e02c4571 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2a26cab97 namespace: kserve-ci-e2e-test resourceVersion: "21635" uid: ad51b418-2458-4b62-83c5-9eb3c6182bd0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2a65da8dd namespace: kserve-ci-e2e-test resourceVersion: "21638" uid: 738cd0ee-628b-46f0-a560-a195284c4948 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:23:27Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:23:27Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2a6e912d1 namespace: kserve-ci-e2e-test resourceVersion: "21639" uid: a6fc0931-984c-4c91-8618-b15be8cdd56a reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:23:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:32:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.45:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:23:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:43Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c680b2d9b6a3e7 namespace: kserve-ci-e2e-test resourceVersion: "26240" uid: 115dd9b0-7273-428d-ab21-eb4ec2d488dc reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c68133b15772d9 namespace: kserve-ci-e2e-test resourceVersion: "26075" uid: 26427782-6074-4191-93e1-c356f16145ca reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c68133b1589429 namespace: kserve-ci-e2e-test resourceVersion: "26078" uid: 50e3f005-fa71-4275-9e3a-046335cd16ab reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:43Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm namespace: kserve-ci-e2e-test resourceVersion: "21555" uid: 3369701d-a56c-44c5-bff1-8ecd97555590 kind: Event lastTimestamp: "2026-07-28T16:32:43Z" message: 'Readiness probe failed: Get "https://10.134.0.45:8643/healthz": dial tcp 10.134.0.45:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:43Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:43Z" name: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm.18c681344e671d62 namespace: kserve-ci-e2e-test resourceVersion: "26239" uid: ac8ced5a-82f1-45a8-9ba0-46b0ff9758c5 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-c3dc2-predictor-58b487689c namespace: kserve-ci-e2e-test resourceVersion: "21537" uid: 67b953df-10b8-4007-8e9a-c7d3c3abd27b kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'Created pod: success-200-isvc-c3dc2-predictor-58b487689c-z5qjm' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2-predictor-58b487689c.18c680b2800d5e60 namespace: kserve-ci-e2e-test resourceVersion: "21554" uid: 8c85932c-b803-4f40-a0e6-0e77b7743672 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21534" uid: 14fc4761-e2bc-4c3c-a8ba-346fa533c49a kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: Scaled up replica set success-200-isvc-c3dc2-predictor-58b487689c from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2-predictor.18c680b27cf6dce5 namespace: kserve-ci-e2e-test resourceVersion: "21542" uid: b9b67db7-bc10-4cca-8b36-24d0988e4556 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: cd6f6500-4b9d-4f95-b270-577ef2dd3717 kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: success-200-isvc-c3dc2-predictor.18c680b5fbe4cd52 namespace: kserve-ci-e2e-test resourceVersion: "21907" uid: 66e112cc-983d-4d69-870b-de4be93451a5 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:23:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21538" uid: cd6f6500-4b9d-4f95-b270-577ef2dd3717 kind: Event lastTimestamp: "2026-07-28T16:23:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:23:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:56Z" name: success-200-isvc-c3dc2-predictor.18c680b5fc6d5bf2 namespace: kserve-ci-e2e-test resourceVersion: "21908" uid: 4b00dfb5-a4c4-4cfc-909d-4ba077052347 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:24:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: cd6f6500-4b9d-4f95-b270-577ef2dd3717 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: success-200-isvc-c3dc2-predictor.18c680bcfbdfc08c namespace: kserve-ci-e2e-test resourceVersion: "22295" uid: 646336ad-ab75-4ec3-a43f-6b3ad8b26ad2 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:24:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-c3dc2-predictor namespace: kserve-ci-e2e-test resourceVersion: "21778" uid: cd6f6500-4b9d-4f95-b270-577ef2dd3717 kind: Event lastTimestamp: "2026-07-28T16:24:41Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:24:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:24:41Z" name: success-200-isvc-c3dc2-predictor.18c680bcfc890594 namespace: kserve-ci-e2e-test resourceVersion: "22296" uid: d46c2046-aab2-4993-ac8f-08c2033ba078 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 7cd9edd1-f5cb-4e93-91ae-31f634c97bd0 kind: Event lastTimestamp: "2026-07-28T16:24:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:14Z" name: success-200-isvc-c3dc2.18c680b27d835431 namespace: kserve-ci-e2e-test resourceVersion: "22050" uid: 1ef8e8e6-f62a-4139-8ad5-ae368b057faf reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 7cd9edd1-f5cb-4e93-91ae-31f634c97bd0 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'Failed to update status for InferenceService "success-200-isvc-c3dc2": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2.18c680b27e3a56b6 namespace: kserve-ci-e2e-test resourceVersion: "21545" uid: 3b2c33d9-4f3d-4ac3-aa0b-ea1fedd6a4e1 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:26Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "21511" uid: 7cd9edd1-f5cb-4e93-91ae-31f634c97bd0 kind: Event lastTimestamp: "2026-07-28T16:23:26Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-c3dc2": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:23:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:26Z" name: success-200-isvc-c3dc2.18c680b27e3ab21d namespace: kserve-ci-e2e-test resourceVersion: "21547" uid: 6c2747c0-0444-4587-af22-6c4d3db54682 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:24:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-c3dc2 namespace: kserve-ci-e2e-test resourceVersion: "22048" uid: 7cd9edd1-f5cb-4e93-91ae-31f634c97bd0 kind: Event lastTimestamp: "2026-07-28T16:24:14Z" message: InferenceService [success-200-isvc-c3dc2] is Ready metadata: creationTimestamp: "2026-07-28T16:24:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:24:14Z" name: success-200-isvc-c3dc2.18c680bd9835e5b2 namespace: kserve-ci-e2e-test resourceVersion: "22049" uid: 820df88c-a072-489a-9ce1-9416e70650b1 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:32:41.405837Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26123" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133b69a0b48 namespace: kserve-ci-e2e-test resourceVersion: "26133" uid: 1b305558-dfa6-4fd2-93b0-7706dc3ee3a1 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26141" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Add eth0 [10.134.0.49/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d0f6fb8b namespace: kserve-ci-e2e-test resourceVersion: "26189" uid: 5f782aa3-db09-4d1a-b965-1733d9d6bbaf reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d25736ac namespace: kserve-ci-e2e-test resourceVersion: "26191" uid: 635d7d2d-3835-4521-adfe-518cc45458e4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d54d0b88 namespace: kserve-ci-e2e-test resourceVersion: "26192" uid: c4e71a5b-7691-4291-b639-3d4878c208f5 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d5c7e233 namespace: kserve-ci-e2e-test resourceVersion: "26193" uid: 3e1826ff-0c5c-472f-8b16-6e768f650bd7 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d5d31417 namespace: kserve-ci-e2e-test resourceVersion: "26194" uid: 626e4d53-a8e5-42b5-9cc6-6c817b6726fb reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d8f61fd0 namespace: kserve-ci-e2e-test resourceVersion: "26197" uid: da761b41-afd5-40c0-b0d6-e20c524431dd reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68133d97bc5f5 namespace: kserve-ci-e2e-test resourceVersion: "26198" uid: 8cf6511d-3b78-462a-9a70-bf70aa6b9b42 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:32:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:33:18Z" message: 'Readiness probe failed: dial tcp 10.134.0.49:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:32:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:18Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c6813405dd7982 namespace: kserve-ci-e2e-test resourceVersion: "26694" uid: 29bc4fed-017b-4e13-ae1f-90b35b783ea1 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:50:00Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:00Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68225bb546aff namespace: kserve-ci-e2e-test resourceVersion: "33792" uid: c693c023-b092-47b5-b53d-7559c7b5f457 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:50:00Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:00Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c68225bb54ee2c namespace: kserve-ci-e2e-test resourceVersion: "33793" uid: 2a4aa610-d19d-495f-8ef8-3ca7b8148d18 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p namespace: kserve-ci-e2e-test resourceVersion: "26129" uid: 6d71f4da-75b7-4681-8b59-daeff6441f33 kind: Event lastTimestamp: "2026-07-28T16:50:03Z" message: 'Readiness probe failed: Get "https://10.134.0.49:8643/healthz": dial tcp 10.134.0.49:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:03Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p.18c682266689fb32 namespace: kserve-ci-e2e-test resourceVersion: "33863" uid: 068948a2-727a-4953-9521-d2eff978d6eb reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-d3a3d-predictor-777b55b9d9 namespace: kserve-ci-e2e-test resourceVersion: "26098" uid: 7cd88e53-39b6-42d0-8e63-1f9169f11344 kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Created pod: success-200-isvc-d3a3d-predictor-777b55b9d9-bc57p' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor-777b55b9d9.18c68133b60785bc namespace: kserve-ci-e2e-test resourceVersion: "26128" uid: ecece72a-7053-4122-9183-98632ffd3e58 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26089" uid: 61c97d8c-81f1-415f-8452-0d9fb9184d4a kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: Scaled up replica set success-200-isvc-d3a3d-predictor-777b55b9d9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d-predictor.18c68133b3f00a6f namespace: kserve-ci-e2e-test resourceVersion: "26104" uid: 4d1c937c-fdb0-408a-9b78-f008ca3c280e reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: adcea179-3e06-49ce-8cc4-1dba1d9195f0 kind: Event lastTimestamp: "2026-07-28T16:33:11Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:11Z" name: success-200-isvc-d3a3d-predictor.18c68137327d7829 namespace: kserve-ci-e2e-test resourceVersion: "26631" uid: f7108a46-60f6-4714-ad44-84c6b861072f reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:32:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26097" uid: adcea179-3e06-49ce-8cc4-1dba1d9195f0 kind: Event lastTimestamp: "2026-07-28T16:33:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:32:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:11Z" name: success-200-isvc-d3a3d-predictor.18c6813732efbd75 namespace: kserve-ci-e2e-test resourceVersion: "26632" uid: 74f0dd8f-0327-43a8-ba0a-03d3dbded191 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: adcea179-3e06-49ce-8cc4-1dba1d9195f0 kind: Event lastTimestamp: "2026-07-28T16:33:56Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:56Z" name: success-200-isvc-d3a3d-predictor.18c6813e309ce96f namespace: kserve-ci-e2e-test resourceVersion: "27058" uid: 3c497678-eb1d-4a43-b709-f2e260b66db4 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:33:26Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-d3a3d-predictor namespace: kserve-ci-e2e-test resourceVersion: "26502" uid: adcea179-3e06-49ce-8cc4-1dba1d9195f0 kind: Event lastTimestamp: "2026-07-28T16:33:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:33:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:56Z" name: success-200-isvc-d3a3d-predictor.18c6813e30e490cf namespace: kserve-ci-e2e-test resourceVersion: "27059" uid: 81145bad-d457-45bc-b4fa-87322c29984a reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: a4fbc80e-78cf-43a9-bc23-670a7f627c1d kind: Event lastTimestamp: "2026-07-28T16:33:28Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:28Z" name: success-200-isvc-d3a3d.18c68133b41f35d2 namespace: kserve-ci-e2e-test resourceVersion: "26786" uid: 55d2648c-c90d-4894-af3d-442bc420efed reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: a4fbc80e-78cf-43a9-bc23-670a7f627c1d kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'Failed to update status for InferenceService "success-200-isvc-d3a3d": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d3a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d.18c68133b4e48fed namespace: kserve-ci-e2e-test resourceVersion: "26121" uid: 3948a896-538e-407a-ae61-afcdf6a6d37b reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:32:41Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26062" uid: a4fbc80e-78cf-43a9-bc23-670a7f627c1d kind: Event lastTimestamp: "2026-07-28T16:32:41Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-d3a3d": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:32:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:32:41Z" name: success-200-isvc-d3a3d.18c68133b4e4e776 namespace: kserve-ci-e2e-test resourceVersion: "26124" uid: 833bad92-b06f-46de-b138-6cd757e82a18 reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:28Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26784" uid: a4fbc80e-78cf-43a9-bc23-670a7f627c1d kind: Event lastTimestamp: "2026-07-28T16:33:28Z" message: InferenceService [success-200-isvc-d3a3d] is Ready metadata: creationTimestamp: "2026-07-28T16:33:28Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:28Z" name: success-200-isvc-d3a3d.18c6813ec21b071e namespace: kserve-ci-e2e-test resourceVersion: "26785" uid: 7804a0eb-7e70-472a-bb6a-c83a46d14405 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:20:56.114700Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19781" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808f7ff9cc74 namespace: kserve-ci-e2e-test resourceVersion: "19796" uid: f5ca47fc-8361-4f42-8477-9c3ae2ba752c reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19803" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Add eth0 [10.134.0.39/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808f9be0703f namespace: kserve-ci-e2e-test resourceVersion: "19860" uid: f7177d64-8407-4c62-a18e-d5782e2801b5 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808f9d5ea469 namespace: kserve-ci-e2e-test resourceVersion: "19863" uid: e2747f8a-3659-43f5-9507-5b351121eff4 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fa19e7f7f namespace: kserve-ci-e2e-test resourceVersion: "19864" uid: 6014b273-9486-4fe1-8bde-19b9048e1f69 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fa2209565 namespace: kserve-ci-e2e-test resourceVersion: "19865" uid: 04267efb-a99f-470d-8918-a645913c560c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fa22cf387 namespace: kserve-ci-e2e-test resourceVersion: "19866" uid: c2e82519-ec3a-4bea-a0f4-424fd4bd17e0 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fa60ce69a namespace: kserve-ci-e2e-test resourceVersion: "19867" uid: cdf1b372-5c4e-4199-8769-07fdbefde6bf reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fa6900e9d namespace: kserve-ci-e2e-test resourceVersion: "19868" uid: b84e3958-1f84-42c9-a271-9e5c0c4a8ddc reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 7 eventTime: null firstTimestamp: "2026-07-28T16:20:57Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:21:43Z" message: 'Readiness probe failed: dial tcp 10.134.0.39:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:20:57Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:21:43Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c6808fbbd7efbf namespace: kserve-ci-e2e-test resourceVersion: "20247" uid: 87cc96c4-e192-4be1-808b-50784ee574ff reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c680a22d0533cd namespace: kserve-ci-e2e-test resourceVersion: "20736" uid: b797b8c9-937a-4f25-985a-e919826e7b5d reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c680a22d05b9e2 namespace: kserve-ci-e2e-test resourceVersion: "20738" uid: ec250b69-9d1d-4ed6-a1c8-9d783fa3df62 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn namespace: kserve-ci-e2e-test resourceVersion: "19787" uid: bd93d12a-16ab-4c47-a9bd-fdf83daf0a86 kind: Event lastTimestamp: "2026-07-28T16:22:18Z" message: 'Readiness probe failed: Get "https://10.134.0.39:8643/healthz": dial tcp 10.134.0.39:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:18Z" name: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn.18c680a29864e1d4 namespace: kserve-ci-e2e-test resourceVersion: "20839" uid: eec2e479-eb8c-44a2-a2ca-ff2dc11481f7 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-daf0b-predictor-5b75479fb9 namespace: kserve-ci-e2e-test resourceVersion: "19771" uid: 67f85d5d-8fc2-4b54-b5da-86fa3aa0236d kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Created pod: success-200-isvc-daf0b-predictor-5b75479fb9-k2bzn' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor-5b75479fb9.18c6808f7f9467d3 namespace: kserve-ci-e2e-test resourceVersion: "19790" uid: 76fa956c-8092-4da2-a697-be2fb3c0dfff reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19769" uid: 7c6def3c-fdf1-4288-995c-ed1e2668879a kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: Scaled up replica set success-200-isvc-daf0b-predictor-5b75479fb9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b-predictor.18c6808f79e55ed8 namespace: kserve-ci-e2e-test resourceVersion: "19772" uid: 8b16be35-c9ab-4d49-89b2-3e6ff5064aec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: ee46b518-1c82-4e72-bfe1-6acfcecfd42b kind: Event lastTimestamp: "2026-07-28T16:21:26Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:21:26Z" name: success-200-isvc-daf0b-predictor.18c68092f865a33f namespace: kserve-ci-e2e-test resourceVersion: "20114" uid: ecbf17a0-13e9-4828-9ab2-e7a7d3682ef7 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 2 eventTime: null firstTimestamp: "2026-07-28T16:21:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "19770" uid: ee46b518-1c82-4e72-bfe1-6acfcecfd42b kind: Event lastTimestamp: "2026-07-28T16:21:26Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:21:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:21:26Z" name: success-200-isvc-daf0b-predictor.18c68092f8da3c7c namespace: kserve-ci-e2e-test resourceVersion: "20115" uid: 31413a5e-e185-40e4-b9fc-54923fbd1305 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ee46b518-1c82-4e72-bfe1-6acfcecfd42b kind: Event lastTimestamp: "2026-07-28T16:22:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:11Z" name: success-200-isvc-daf0b-predictor.18c68099f6744ecf namespace: kserve-ci-e2e-test resourceVersion: "20651" uid: d8550946-e488-4837-974e-af4d0912c99a reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:21:41Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-daf0b-predictor namespace: kserve-ci-e2e-test resourceVersion: "20002" uid: ee46b518-1c82-4e72-bfe1-6acfcecfd42b kind: Event lastTimestamp: "2026-07-28T16:22:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:21:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:11Z" name: success-200-isvc-daf0b-predictor.18c68099f6eaa2e8 namespace: kserve-ci-e2e-test resourceVersion: "20652" uid: 80130fdf-e982-4136-aa83-c3e1b4548154 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 5 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 31db4f93-0583-4025-9c12-f7688cef9cdc kind: Event lastTimestamp: "2026-07-28T16:21:53Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:21:53Z" name: success-200-isvc-daf0b.18c6808f79ef9315 namespace: kserve-ci-e2e-test resourceVersion: "20331" uid: b3cd68fc-4f95-4281-b18f-1a175005f5a6 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 31db4f93-0583-4025-9c12-f7688cef9cdc kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'Failed to update status for InferenceService "success-200-isvc-daf0b": Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b.18c6808f7a8289a2 namespace: kserve-ci-e2e-test resourceVersion: "19775" uid: 6b13bbf6-d314-4056-b1cf-c5c33f40b9a5 reason: UpdateFailed reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:56Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "19761" uid: 31db4f93-0583-4025-9c12-f7688cef9cdc kind: Event lastTimestamp: "2026-07-28T16:20:56Z" message: 'fails to update InferenceService status: Operation cannot be fulfilled on inferenceservices.serving.kserve.io "success-200-isvc-daf0b": the object has been modified; please apply your changes to the latest version and try again' metadata: creationTimestamp: "2026-07-28T16:20:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:20:56Z" name: success-200-isvc-daf0b.18c6808f7a82f24c namespace: kserve-ci-e2e-test resourceVersion: "19777" uid: 4f29391c-b048-4e8e-b48d-9ab740bfe7ce reason: InternalError reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:21:53Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-daf0b namespace: kserve-ci-e2e-test resourceVersion: "20328" uid: 31db4f93-0583-4025-9c12-f7688cef9cdc kind: Event lastTimestamp: "2026-07-28T16:21:53Z" message: InferenceService [success-200-isvc-daf0b] is Ready metadata: creationTimestamp: "2026-07-28T16:21:53Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:21:53Z" name: success-200-isvc-daf0b.18c6809cccb3c350 namespace: kserve-ci-e2e-test resourceVersion: "20329" uid: 63d59ca6-97e4-42e0-8ca6-af2fa2d25b19 reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:22:16.389623Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20756" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a230bb7fb7 namespace: kserve-ci-e2e-test resourceVersion: "20764" uid: 46477a8c-f542-42ea-b95d-181833009b0f reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "success-200-isvc-e5aa9-predictor-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a23ba2694f namespace: kserve-ci-e2e-test resourceVersion: "20800" uid: 40583cd5-afac-471e-bef4-7750720e3b19 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20773" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Add eth0 [10.134.0.43/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a26f5e53c0 namespace: kserve-ci-e2e-test resourceVersion: "20820" uid: 0c53c65c-01ed-4df5-b691-4e5e601506c2 reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Container image "quay.io/opendatahub/success-200-isvc:odh-pr-1821" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a270d50527 namespace: kserve-ci-e2e-test resourceVersion: "20822" uid: e3f88f21-4f93-4834-b492-96ee371b5e23 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: 'Created container: kserve-container' metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a274f1b527 namespace: kserve-ci-e2e-test resourceVersion: "20823" uid: bb10b7f5-f84e-43b2-8ed5-3a1b4879ea3d reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Started container kserve-container metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a2759a3ecf namespace: kserve-ci-e2e-test resourceVersion: "20824" uid: cdfdf1db-a653-4d80-b8ce-d312f8dbdd63 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Container image "quay.io/opendatahub/odh-kube-auth-proxy@sha256:dcb09fbabd8811f0956ef612a0c9ddd5236804b9bd6548a0647d2b531c9d01b3" already present on machine metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a275aff625 namespace: kserve-ci-e2e-test resourceVersion: "20825" uid: f14cc2c4-fcc0-483a-8b2a-9339303a0d07 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: 'Created container: kube-rbac-proxy' metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a278caa23d namespace: kserve-ci-e2e-test resourceVersion: "20829" uid: f478759e-a388-4f72-a5b3-4b12d6dd15d6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:17Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:22:17Z" message: Started container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:22:17Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:22:17Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a2794d9ac6 namespace: kserve-ci-e2e-test resourceVersion: "20830" uid: d8212722-8ac6-467c-9fb4-4df02dcbcf68 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 8 eventTime: null firstTimestamp: "2026-07-28T16:22:18Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:31:44Z" message: 'Readiness probe failed: dial tcp 10.134.0.43:8080: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:22:18Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:44Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c680a2b3133688 namespace: kserve-ci-e2e-test resourceVersion: "25304" uid: 478e3c23-10e6-42fb-aefa-229498bff57a reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kserve-container} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Stopping container kserve-container metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c68125be932b39 namespace: kserve-ci-e2e-test resourceVersion: "25205" uid: c5356387-1be2-4691-babd-2a0c8cf1e64f reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:31:41Z" message: Stopping container kube-rbac-proxy metadata: creationTimestamp: "2026-07-28T16:31:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:41Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c68125be944d14 namespace: kserve-ci-e2e-test resourceVersion: "25209" uid: 10197a64-b909-415c-99fd-379e15a6ade1 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:31:44Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{kube-rbac-proxy} kind: Pod name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l namespace: kserve-ci-e2e-test resourceVersion: "20759" uid: af2e7a73-8319-4118-a799-4d8a2fdfd4e7 kind: Event lastTimestamp: "2026-07-28T16:31:44Z" message: 'Readiness probe failed: Get "https://10.134.0.43:8643/healthz": dial tcp 10.134.0.43:8643: connect: connection refused' metadata: creationTimestamp: "2026-07-28T16:31:44Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:31:44Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l.18c681267bac3447 namespace: kserve-ci-e2e-test resourceVersion: "25303" uid: 3497e6e1-41f1-412d-bf28-1c3fe9e8ec4f reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: success-200-isvc-e5aa9-predictor-6d94bd5dd5 namespace: kserve-ci-e2e-test resourceVersion: "20737" uid: aa588b11-ee71-4a1e-94f0-5e1f4128dde9 kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: 'Created pod: success-200-isvc-e5aa9-predictor-6d94bd5dd5-7g25l' metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-e5aa9-predictor-6d94bd5dd5.18c680a23025eb02 namespace: kserve-ci-e2e-test resourceVersion: "20758" uid: f116ecc9-736a-463f-828a-9addf80f99f5 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: success-200-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20734" uid: 926015e3-79c8-4444-b0de-6a642b5fd7ff kind: Event lastTimestamp: "2026-07-28T16:22:16Z" message: Scaled up replica set success-200-isvc-e5aa9-predictor-6d94bd5dd5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:22:16Z" name: success-200-isvc-e5aa9-predictor.18c680a22d723314 namespace: kserve-ci-e2e-test resourceVersion: "20739" uid: a9a5e421-ea5e-4dfc-a695-eb3de36874ec reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 64fcdff7-539f-45dc-904f-b3e3da9b3696 kind: Event lastTimestamp: "2026-07-28T16:23:01Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:01Z" name: success-200-isvc-e5aa9-predictor.18c680a5ac3bd4b6 namespace: kserve-ci-e2e-test resourceVersion: "21216" uid: 0f8bb858-df0c-4b07-bd1f-266b9d94a926 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:22:31Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20735" uid: 64fcdff7-539f-45dc-904f-b3e3da9b3696 kind: Event lastTimestamp: "2026-07-28T16:23:01Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:22:31Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:01Z" name: success-200-isvc-e5aa9-predictor.18c680a5acdc5da6 namespace: kserve-ci-e2e-test resourceVersion: "21217" uid: 3a3e5a64-315a-4e7b-93b8-de78f5beb7f4 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 64fcdff7-539f-45dc-904f-b3e3da9b3696 kind: Event lastTimestamp: "2026-07-28T16:23:46Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:46Z" name: success-200-isvc-e5aa9-predictor.18c680b029ae4fec namespace: kserve-ci-e2e-test resourceVersion: "21827" uid: b19aae01-1a7c-4425-8f5d-32883a9bbf3b reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 3 eventTime: null firstTimestamp: "2026-07-28T16:23:16Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: success-200-isvc-e5aa9-predictor namespace: kserve-ci-e2e-test resourceVersion: "20970" uid: 64fcdff7-539f-45dc-904f-b3e3da9b3696 kind: Event lastTimestamp: "2026-07-28T16:23:46Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:23:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:23:46Z" name: success-200-isvc-e5aa9-predictor.18c680b02a1cf4a4 namespace: kserve-ci-e2e-test resourceVersion: "21828" uid: aa509e73-298f-4857-998e-f120fc5ff3e8 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:22:16Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "20716" uid: 5025cb74-fdf7-4489-baa3-d4d7592821e1 kind: Event lastTimestamp: "2026-07-28T16:23:14Z" message: Istio VirtualService CRD not present; VirtualService reconciliation skipped. If you do not use Istio, set ingress.disableIstioVirtualHost=true. metadata: creationTimestamp: "2026-07-28T16:22:16Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:14Z" name: success-200-isvc-e5aa9.18c680a22d7b8292 namespace: kserve-ci-e2e-test resourceVersion: "21336" uid: 161f99f6-8392-413b-9e0a-9bd9c1129005 reason: VirtualServiceCRDNotFound reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:23:14Z" involvedObject: apiVersion: serving.kserve.io/v1beta1 kind: InferenceService name: success-200-isvc-e5aa9 namespace: kserve-ci-e2e-test resourceVersion: "21334" uid: 5025cb74-fdf7-4489-baa3-d4d7592821e1 kind: Event lastTimestamp: "2026-07-28T16:23:14Z" message: InferenceService [success-200-isvc-e5aa9] is Ready metadata: creationTimestamp: "2026-07-28T16:23:14Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:23:14Z" name: success-200-isvc-e5aa9.18c680afc4fa40d0 namespace: kserve-ci-e2e-test resourceVersion: "21335" uid: 4067d4eb-afa4-4d6a-8f3a-25af27f6ac1d reason: InferenceServiceReady reportingComponent: v1beta1Controllers reportingInstance: "" source: component: v1beta1Controllers type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:12:41.309220Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16061" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-021cc-6486fdbd5-9fr9n to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6801c4b44ec3a namespace: kserve-ci-e2e-test resourceVersion: "16066" uid: 5305b4ff-3c92-4376-9a8e-60bfd575c5e8 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16070" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: Add eth0 [10.134.0.36/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6801c6531eab6 namespace: kserve-ci-e2e-test resourceVersion: "16082" uid: 2673d0ae-04d4-4702-a9ff-8dc0208d207e reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-021cc} kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6801c66de1c5b namespace: kserve-ci-e2e-test resourceVersion: "16085" uid: 176ce3de-1670-4fe3-b040-fe413940509c reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-021cc} kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: 'Created container: switch-graph-021cc' metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6801c6a9a9fba namespace: kserve-ci-e2e-test resourceVersion: "16086" uid: a7673b06-d791-4c18-8b0f-f488922e9255 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-021cc} kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: Started container switch-graph-021cc metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6801c6b59f72a namespace: kserve-ci-e2e-test resourceVersion: "16087" uid: d4643d02-3606-429b-975b-192f134dfe8c reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:20:55Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-021cc} kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:20:55Z" message: Stopping container switch-graph-021cc metadata: creationTimestamp: "2026-07-28T16:20:55Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:20:55Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c6808f6d187e06 namespace: kserve-ci-e2e-test resourceVersion: "19755" uid: 1dade71f-cdbd-4575-b2c6-6bfbdeddc683 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:20:58Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-021cc} kind: Pod name: switch-graph-021cc-6486fdbd5-9fr9n namespace: kserve-ci-e2e-test resourceVersion: "16062" uid: 15a1d87c-0187-4d95-b98b-9ecd12661654 kind: Event lastTimestamp: "2026-07-28T16:21:23Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:20:58Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:21:23Z" name: switch-graph-021cc-6486fdbd5-9fr9n.18c680901bf6dbaf namespace: kserve-ci-e2e-test resourceVersion: "20097" uid: b52788b1-6d57-411d-bc35-ad5df7584fc4 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-021cc-6486fdbd5 namespace: kserve-ci-e2e-test resourceVersion: "16058" uid: 0964f36d-fd75-4766-b960-83c425a84558 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: 'Created pod: switch-graph-021cc-6486fdbd5-9fr9n' metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc-6486fdbd5.18c6801c4ad40e95 namespace: kserve-ci-e2e-test resourceVersion: "16063" uid: b69a74f9-9d99-46b4-9801-a2b4d64897a1 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16056" uid: 00fa3084-34c2-4217-8c92-f26a1dc7aa44 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: Scaled up replica set switch-graph-021cc-6486fdbd5 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc.18c6801c49e3a9c9 namespace: kserve-ci-e2e-test resourceVersion: "16060" uid: fe64087b-73d4-400c-a45c-b70fef91244d reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16074" uid: f24c82f0-161c-4523-944b-f6f8e6874cd8 kind: Event lastTimestamp: "2026-07-28T16:12:41Z" message: InferenceGraph [switch-graph-021cc] is Ready metadata: creationTimestamp: "2026-07-28T16:12:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:12:41Z" name: switch-graph-021cc.18c6801c500f001e namespace: kserve-ci-e2e-test resourceVersion: "16076" uid: db5c7784-5e63-447b-bfcf-c5d4e2195cc0 reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 6a06c731-beb0-4703-9d84-bc0f575d7f6c kind: Event lastTimestamp: "2026-07-28T16:12:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:56Z" name: switch-graph-021cc.18c6801fc7f791e4 namespace: kserve-ci-e2e-test resourceVersion: "16360" uid: 1e4dc57f-e53c-405b-8ff8-c8f7a7c55903 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:12:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16057" uid: 6a06c731-beb0-4703-9d84-bc0f575d7f6c kind: Event lastTimestamp: "2026-07-28T16:12:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:12:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:12:56Z" name: switch-graph-021cc.18c6801fc845b984 namespace: kserve-ci-e2e-test resourceVersion: "16363" uid: d745d243-b375-43eb-9812-7776d22cf6ad reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 6a06c731-beb0-4703-9d84-bc0f575d7f6c kind: Event lastTimestamp: "2026-07-28T16:13:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:11Z" name: switch-graph-021cc.18c680234757c4bb namespace: kserve-ci-e2e-test resourceVersion: "16491" uid: 6e55be30-38c2-4083-ab9c-32a812060441 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:13:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-021cc namespace: kserve-ci-e2e-test resourceVersion: "16361" uid: 6a06c731-beb0-4703-9d84-bc0f575d7f6c kind: Event lastTimestamp: "2026-07-28T16:13:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:13:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:13:11Z" name: switch-graph-021cc.18c68023484bbf1d namespace: kserve-ci-e2e-test resourceVersion: "16492" uid: 2c55059e-2335-4452-99c6-e6502358991b reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:11:26.896157Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15207" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-3692d-d8b5895b7-wbvlr to ip-10-0-142-124.ec2.internal metadata: creationTimestamp: "2026-07-28T16:11:26Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:11:26Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800af7e7a6ab namespace: kserve-ci-e2e-test resourceVersion: "15214" uid: 66f74191-4aa9-4648-8241-66618575b214 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:27Z" message: 'MountVolume.SetUp failed for volume "proxy-tls" : secret "switch-graph-3692d-serving-cert" not found' metadata: creationTimestamp: "2026-07-28T16:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:27Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800afe792a7e namespace: kserve-ci-e2e-test resourceVersion: "15222" uid: 7e1bb79c-53d2-4fb0-911e-ef45717f54b6 reason: FailedMount reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:27Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15216" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:27Z" message: Add eth0 [10.134.0.32/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:11:27Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800b35d4cd32 namespace: kserve-ci-e2e-test resourceVersion: "15232" uid: b567e610-c278-49e3-9ccb-e85baad1811f reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:27Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:27Z" message: Pulling image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" metadata: creationTimestamp: "2026-07-28T16:11:27Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:27Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800b37220a23 namespace: kserve-ci-e2e-test resourceVersion: "15234" uid: 7c6ffcaa-a812-4d41-8be5-0beb2204dcaf reason: Pulling reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:30Z" message: 'Successfully pulled image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" in 2.794s (2.794s including waiting). Image size: 218225243 bytes.' metadata: creationTimestamp: "2026-07-28T16:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:30Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800bddb83027 namespace: kserve-ci-e2e-test resourceVersion: "15255" uid: 826932b0-9a70-408b-bf90-56b177e79df2 reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:30Z" message: 'Created container: switch-graph-3692d' metadata: creationTimestamp: "2026-07-28T16:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:30Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800be0c5a920 namespace: kserve-ci-e2e-test resourceVersion: "15257" uid: 7f4476de-bd4f-4f36-8b58-eb5e1b054386 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:30Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:30Z" message: Started container switch-graph-3692d metadata: creationTimestamp: "2026-07-28T16:11:30Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:30Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800be142d346 namespace: kserve-ci-e2e-test resourceVersion: "15258" uid: 2fde1829-32b0-46e3-a055-13848055a3f6 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:11:41Z" message: Stopping container switch-graph-3692d metadata: creationTimestamp: "2026-07-28T16:11:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:11:41Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800e41d65d7e namespace: kserve-ci-e2e-test resourceVersion: "15367" uid: 75826e9e-011f-44c6-b1c1-689ebebda039 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:11:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-3692d} kind: Pod name: switch-graph-3692d-d8b5895b7-wbvlr namespace: kserve-ci-e2e-test resourceVersion: "15210" uid: 6a379b69-6194-4300-be5f-dc427636e814 kind: Event lastTimestamp: "2026-07-28T16:12:07Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:11:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:12:07Z" name: switch-graph-3692d-d8b5895b7-wbvlr.18c6800e91c3070f namespace: kserve-ci-e2e-test resourceVersion: "15730" uid: 32b90b72-3391-466f-968c-3cb5a2b01729 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-142-124.ec2.internal source: component: kubelet host: ip-10-0-142-124.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:26Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-3692d-d8b5895b7 namespace: kserve-ci-e2e-test resourceVersion: "15204" uid: 8db150e0-4989-4adc-83c4-4a637ba5a046 kind: Event lastTimestamp: "2026-07-28T16:11:26Z" message: 'Created pod: switch-graph-3692d-d8b5895b7-wbvlr' metadata: creationTimestamp: "2026-07-28T16:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:26Z" name: switch-graph-3692d-d8b5895b7.18c6800af7348ef9 namespace: kserve-ci-e2e-test resourceVersion: "15209" uid: cc0f2b64-b854-4c2c-8edc-d7a4a8b63e34 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:26Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-3692d namespace: kserve-ci-e2e-test resourceVersion: "15202" uid: 7fe435cd-9948-4dd4-b0e1-aa1389ee0444 kind: Event lastTimestamp: "2026-07-28T16:11:26Z" message: Scaled up replica set switch-graph-3692d-d8b5895b7 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:11:26Z" name: switch-graph-3692d.18c6800af62cd288 namespace: kserve-ci-e2e-test resourceVersion: "15205" uid: 26fcb68b-06c5-4973-af7e-af10660c1218 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:11:26Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-3692d namespace: kserve-ci-e2e-test resourceVersion: "15219" uid: daecad7f-fb20-4303-9480-800ee21de0a5 kind: Event lastTimestamp: "2026-07-28T16:11:26Z" message: InferenceGraph [switch-graph-3692d] is Ready metadata: creationTimestamp: "2026-07-28T16:11:26Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:11:26Z" name: switch-graph-3692d.18c6800afc97ddb0 namespace: kserve-ci-e2e-test resourceVersion: "15221" uid: 9fe15d3b-0ea7-47ba-9743-b52f34af9e3f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - action: Binding apiVersion: v1 eventTime: "2026-07-28T16:33:41.530855Z" firstTimestamp: null involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26914" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: null message: Successfully assigned kserve-ci-e2e-test/switch-graph-d3a3d-6fc78c5b9-pzf8m to ip-10-0-140-224.ec2.internal metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: events.k8s.io/v1 fieldsType: FieldsV1 fieldsV1: f:action: {} f:eventTime: {} f:note: {} f:reason: {} f:regarding: {} f:reportingController: {} f:reportingInstance: {} f:type: {} manager: kube-scheduler operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68141b65507ab namespace: kserve-ci-e2e-test resourceVersion: "26918" uid: e2fc564d-fe12-488b-9aa4-9d35af8ce537 reason: Scheduled reportingComponent: default-scheduler reportingInstance: default-scheduler-kube-scheduler-54d798bcf5-lh9d6 source: {} type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: v1 kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26923" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:33:41Z" message: Add eth0 [10.133.0.28/23] from ovn-kubernetes metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: multus-daemon operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68141d0a9e21f namespace: kserve-ci-e2e-test resourceVersion: "26935" uid: 452757c1-0f1b-420c-af47-15723f39355d reason: AddedInterface reportingComponent: multus reportingInstance: "" source: component: multus type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d3a3d} kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:33:41Z" message: Container image "quay.io/opendatahub/kserve-router@sha256:ecd8623c42b521978afd08b50dfd4d13bfe521fd8131429e4f24493619574031" already present on machine metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68141d2042492 namespace: kserve-ci-e2e-test resourceVersion: "26937" uid: e257a468-73ae-4084-b91a-0f027b91007e reason: Pulled reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d3a3d} kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:33:42Z" message: 'Created container: switch-graph-d3a3d' metadata: creationTimestamp: "2026-07-28T16:33:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:42Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68141d514a21f namespace: kserve-ci-e2e-test resourceVersion: "26939" uid: 61b67718-fd55-431d-94c9-fd43c50c4ff6 reason: Created reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:42Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d3a3d} kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:33:42Z" message: Started container switch-graph-d3a3d metadata: creationTimestamp: "2026-07-28T16:33:42Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:33:42Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68141d58e81e4 namespace: kserve-ci-e2e-test resourceVersion: "26940" uid: 726371bf-5223-4a9f-89a3-79aa3fa469c4 reason: Started reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:50:00Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d3a3d} kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:50:00Z" message: Stopping container switch-graph-d3a3d metadata: creationTimestamp: "2026-07-28T16:50:00Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:00Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c68225b0783ed5 namespace: kserve-ci-e2e-test resourceVersion: "33774" uid: c6a24ef1-d9b7-43f5-ba67-269a94f37cd8 reason: Killing reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Normal - apiVersion: v1 count: 6 eventTime: null firstTimestamp: "2026-07-28T16:50:03Z" involvedObject: apiVersion: v1 fieldPath: spec.containers{switch-graph-d3a3d} kind: Pod name: switch-graph-d3a3d-6fc78c5b9-pzf8m namespace: kserve-ci-e2e-test resourceVersion: "26916" uid: 64a0de90-5b46-4266-a9db-34fe621656e9 kind: Event lastTimestamp: "2026-07-28T16:50:28Z" message: 'Readiness probe failed: HTTP probe failed with statuscode: 503' metadata: creationTimestamp: "2026-07-28T16:50:03Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:reportingInstance: {} f:source: f:component: {} f:host: {} f:type: {} manager: kubelet operation: Update time: "2026-07-28T16:50:28Z" name: switch-graph-d3a3d-6fc78c5b9-pzf8m.18c682266d184f90 namespace: kserve-ci-e2e-test resourceVersion: "34052" uid: df132be4-0570-4f14-94a9-19c040510dd2 reason: Unhealthy reportingComponent: kubelet reportingInstance: ip-10-0-140-224.ec2.internal source: component: kubelet host: ip-10-0-140-224.ec2.internal type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: apps/v1 kind: ReplicaSet name: switch-graph-d3a3d-6fc78c5b9 namespace: kserve-ci-e2e-test resourceVersion: "26909" uid: 3dd8a63d-2c8b-4000-851c-6731f6cfe83d kind: Event lastTimestamp: "2026-07-28T16:33:41Z" message: 'Created pod: switch-graph-d3a3d-6fc78c5b9-pzf8m' metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d-6fc78c5b9.18c68141b5dc5a61 namespace: kserve-ci-e2e-test resourceVersion: "26915" uid: 86543311-4232-4273-9b4f-3f9015faff89 reason: SuccessfulCreate reportingComponent: replicaset-controller reportingInstance: "" source: component: replicaset-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: apps/v1 kind: Deployment name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26906" uid: 672017dc-b16d-4616-977d-246cf517b740 kind: Event lastTimestamp: "2026-07-28T16:33:41Z" message: Scaled up replica set switch-graph-d3a3d-6fc78c5b9 from 0 to 1 metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d.18c68141b3869397 namespace: kserve-ci-e2e-test resourceVersion: "26911" uid: 90591f80-9ce0-4c25-be1e-f836bc1311a4 reason: ScalingReplicaSet reportingComponent: deployment-controller reportingInstance: "" source: component: deployment-controller type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:41Z" involvedObject: apiVersion: serving.kserve.io/v1alpha1 kind: InferenceGraph name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26926" uid: 03a42e12-eafb-4c05-aeee-449c2b3779d4 kind: Event lastTimestamp: "2026-07-28T16:33:41Z" message: InferenceGraph [switch-graph-d3a3d] is Ready metadata: creationTimestamp: "2026-07-28T16:33:41Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: manager operation: Update time: "2026-07-28T16:33:41Z" name: switch-graph-d3a3d.18c68141ba41860a namespace: kserve-ci-e2e-test resourceVersion: "26929" uid: 292b754f-b517-46b0-b0a6-b5bfec94169f reason: InferenceGraphReady reportingComponent: InferenceGraphController reportingInstance: "" source: component: InferenceGraphController type: Normal - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 16d15868-34d7-45cd-999d-922295e74487 kind: Event lastTimestamp: "2026-07-28T16:33:56Z" message: 'failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:56Z" name: switch-graph-d3a3d.18c681453187e39f namespace: kserve-ci-e2e-test resourceVersion: "27061" uid: 4bf6a7a4-ea3c-41b8-8934-1c0789741c71 reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:33:56Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "26908" uid: 16d15868-34d7-45cd-999d-922295e74487 kind: Event lastTimestamp: "2026-07-28T16:33:56Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: unable to get metrics for resource cpu: no metrics returned from resource metrics API' metadata: creationTimestamp: "2026-07-28T16:33:56Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:33:56Z" name: switch-graph-d3a3d.18c6814531cf4581 namespace: kserve-ci-e2e-test resourceVersion: "27063" uid: 6ae08938-4efa-45fe-86a8-ace856975d33 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: 16d15868-34d7-45cd-999d-922295e74487 kind: Event lastTimestamp: "2026-07-28T16:34:11Z" message: 'failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:11Z" name: switch-graph-d3a3d.18c68148b0ba6d59 namespace: kserve-ci-e2e-test resourceVersion: "27214" uid: 2ae9ebb6-a651-4583-a3ec-69bb6c82ca7e reason: FailedGetResourceMetric reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning - apiVersion: v1 count: 1 eventTime: null firstTimestamp: "2026-07-28T16:34:11Z" involvedObject: apiVersion: autoscaling/v2 kind: HorizontalPodAutoscaler name: switch-graph-d3a3d namespace: kserve-ci-e2e-test resourceVersion: "27062" uid: 16d15868-34d7-45cd-999d-922295e74487 kind: Event lastTimestamp: "2026-07-28T16:34:11Z" message: 'invalid metrics (1 invalid out of 1), first error is: failed to get cpu resource metric value: failed to get cpu utilization: did not receive metrics for targeted pods (pods might be unready)' metadata: creationTimestamp: "2026-07-28T16:34:11Z" managedFields: - apiVersion: v1 fieldsType: FieldsV1 fieldsV1: f:count: {} f:firstTimestamp: {} f:involvedObject: {} f:lastTimestamp: {} f:message: {} f:reason: {} f:reportingComponent: {} f:source: f:component: {} f:type: {} manager: kube-controller-manager operation: Update time: "2026-07-28T16:34:11Z" name: switch-graph-d3a3d.18c68148b1005dec namespace: kserve-ci-e2e-test resourceVersion: "27216" uid: 7683be39-88b2-4ded-bdaf-e85d9bd11010 reason: FailedComputeMetricsReplicas reportingComponent: horizontal-pod-autoscaler reportingInstance: "" source: component: horizontal-pod-autoscaler type: Warning kind: EventList metadata: resourceVersion: "34054"